actions/runner
1
0
Fork 0
mirror of https://github.com/actions/runner.git synced 2025-12-25 02:47:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update Runner to send step updates to Results (#2510)

Browse Source 
* Also send Steps update to Results service

* Refactor to separate results server from current job server

* If hit any error while uploading to Results, skip Results upload

* Add proxy authentication and buffer request for WinHttpHandler

* Remove unnecessary null guard

* Also send Results telemetry when step update fails

* IResultsServer is not disposable
This commit is contained in:
Yang Cao
2023-04-03 16:31:13 -04:00
committed by GitHub
parent 1ceb1a67f2
commit 0484afeec7
9 changed files with 314 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/Sdk/Common/Common/RawClientHttpRequestSettings.cs
View File

@@ -184,9 +184,33 @@ namespace GitHub.Services.Common
return settings;
}
/// <summary>
/// Gets or sets the maximum size allowed for response content buffering.
/// </summary>
[DefaultValue(c_defaultContentBufferSize)]
public Int32 MaxContentBufferSize
{
get
{
return m_maxContentBufferSize;
}
set
{
ArgumentUtility.CheckForOutOfRange(value, nameof(value), 0, c_maxAllowedContentBufferSize);
m_maxContentBufferSize = value;
}
}
private static Lazy<RawClientHttpRequestSettings> s_defaultSettings
= new Lazy<RawClientHttpRequestSettings>(ConstructDefaultSettings);
private Int32 m_maxContentBufferSize;
// We will buffer a maximum of 1024MB in the message handler
private const Int32 c_maxAllowedContentBufferSize = 1024 * 1024 * 1024;
// We will buffer, by default, up to 512MB in the message handler
private const Int32 c_defaultContentBufferSize = 1024 * 1024 * 512;
private const Int32 c_defaultMaxRetry = 3;
private static readonly TimeSpan s_defaultTimeout = TimeSpan.FromSeconds(100); //default WebAPI timeout
private ICollection<CultureInfo> m_acceptLanguages = new List<CultureInfo>();

43
src/Sdk/Common/Common/RawHttpMessageHandler.cs
View File

@@ -9,7 +9,7 @@ using GitHub.Services.OAuth;
namespace GitHub.Services.Common
{
public class RawHttpMessageHandler: HttpMessageHandler
public class RawHttpMessageHandler : HttpMessageHandler
{
public RawHttpMessageHandler(
FederatedCredential credentials)
@@ -120,6 +120,7 @@ namespace GitHub.Services.Common
Boolean succeeded = false;
HttpResponseMessageWrapper responseWrapper;
Boolean lastResponseDemandedProxyAuth = false;
Int32 retries = m_maxAuthRetries;
try
{
@@ -138,7 +139,13 @@ namespace GitHub.Services.Common
// Let's start with sending a token
IssuedToken token = await m_tokenProvider.GetTokenAsync(null, tokenSource.Token).ConfigureAwait(false);
ApplyToken(request, token);
ApplyToken(request, token, applyICredentialsToWebProxy: lastResponseDemandedProxyAuth);
// The WinHttpHandler will chunk any content that does not have a computed length which is
// not what we want. By loading into a buffer up-front we bypass this behavior and there is
// no difference in the normal HttpClientHandler behavior here since this is what they were
// already doing.
await BufferRequestContentAsync(request, tokenSource.Token).ConfigureAwait(false);
// ConfigureAwait(false) enables the continuation to be run outside any captured
// SyncronizationContext (such as ASP.NET's) which keeps things from deadlocking...
@@ -147,7 +154,8 @@ namespace GitHub.Services.Common
responseWrapper = new HttpResponseMessageWrapper(response);
var isUnAuthorized = responseWrapper.StatusCode == HttpStatusCode.Unauthorized;
if (!isUnAuthorized)
lastResponseDemandedProxyAuth = responseWrapper.StatusCode == HttpStatusCode.ProxyAuthenticationRequired;
if (!isUnAuthorized && !lastResponseDemandedProxyAuth)
{
// Validate the token after it has been successfully authenticated with the server.
m_tokenProvider?.ValidateToken(token, responseWrapper);
@@ -211,15 +219,42 @@ namespace GitHub.Services.Common
}
}
private static async Task BufferRequestContentAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
if (request.Content != null &&
request.Headers.TransferEncodingChunked != true)
{
Int64? contentLength = request.Content.Headers.ContentLength;
if (contentLength == null)
{
await request.Content.LoadIntoBufferAsync().EnforceCancellation(cancellationToken).ConfigureAwait(false);
}
// Explicitly turn off chunked encoding since we have computed the request content size
request.Headers.TransferEncodingChunked = false;
}
}
private void ApplyToken(
HttpRequestMessage request,
IssuedToken token)
IssuedToken token,
bool applyICredentialsToWebProxy = false)
{
switch (token)
{
case null:
return;
case ICredentials credentialsToken:
if (applyICredentialsToWebProxy)
{
HttpClientHandler httpClientHandler = m_transportHandler as HttpClientHandler;
if (httpClientHandler != null && httpClientHandler.Proxy != null)
{
httpClientHandler.Proxy.Credentials = credentialsToken;
}
}
m_credentialWrapper.InnerCredentials = credentialsToken;
break;
default:

2
src/Sdk/Common/Common/VssHttpMessageHandler.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;

2
src/Sdk/Common/Common/VssHttpRequestSettings.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Globalization;

41
src/Sdk/WebApi/WebApi/Contracts.cs
View File

@@ -1,3 +1,4 @@
using System.Collections.Generic;
using System.Runtime.Serialization;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
@@ -126,6 +127,46 @@ namespace GitHub.Services.Results.Contracts
public bool Ok;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class StepsUpdateRequest
{
[DataMember]
public IEnumerable<Step> Steps;
[DataMember]
public long ChangeOrder;
[DataMember]
public string WorkflowJobRunBackendId;
[DataMember]
public string WorkflowRunBackendId;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class Step
{
[DataMember]
public string ExternalId;
[DataMember]
public int Number;
[DataMember]
public string Name;
[DataMember]
public Status Status;
[DataMember]
public string StartedAt;
[DataMember]
public string CompletedAt;
}
public enum Status
{
StatusUnknown = 0,
StatusInProgress = 3,
StatusPending = 5,
StatusCompleted = 6
}
public static class BlobStorageTypes
{
public static readonly string AzureBlobStorage = "BLOB_STORAGE_TYPE_AZURE";

74
src/Sdk/WebApi/WebApi/ResultsHttpClient.cs
View File

@@ -1,11 +1,16 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Services.Results.Contracts;
using System.Net.Http.Formatting;
using GitHub.DistributedTask.WebApi;
using GitHub.Services.Common;
using GitHub.Services.Results.Contracts;
using Sdk.WebApi.WebApi;
namespace GitHub.Services.Results.Client
@@ -22,6 +27,7 @@ namespace GitHub.Services.Results.Client
m_token = token;
m_resultsServiceUrl = baseUrl;
m_formatter = new JsonMediaTypeFormatter();
m_changeIdCounter = 1;
}
// Get Sas URL calls
@@ -86,7 +92,7 @@ namespace GitHub.Services.Results.Client
// Create metadata calls
private async Task CreateMetadata<R>(Uri uri, CancellationToken cancellationToken, R request, string timestamp)
private async Task SendRequest<R>(Uri uri, CancellationToken cancellationToken, R request, string timestamp)
{
using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, uri))
{
@@ -121,7 +127,7 @@ namespace GitHub.Services.Results.Client
};
var createStepSummaryMetadataEndpoint = new Uri(m_resultsServiceUrl, Constants.CreateStepSummaryMetadata);
await CreateMetadata<StepSummaryMetadataCreate>(createStepSummaryMetadataEndpoint, cancellationToken, request, timestamp);
await SendRequest<StepSummaryMetadataCreate>(createStepSummaryMetadataEndpoint, cancellationToken, request, timestamp);
}
private async Task StepLogUploadCompleteAsync(string planId, string jobId, Guid stepId, long lineCount, CancellationToken cancellationToken)
@@ -137,7 +143,7 @@ namespace GitHub.Services.Results.Client
};
var createStepLogsMetadataEndpoint = new Uri(m_resultsServiceUrl, Constants.CreateStepLogsMetadata);
await CreateMetadata<StepLogsMetadataCreate>(createStepLogsMetadataEndpoint, cancellationToken, request, timestamp);
await SendRequest<StepLogsMetadataCreate>(createStepLogsMetadataEndpoint, cancellationToken, request, timestamp);
}
private async Task JobLogUploadCompleteAsync(string planId, string jobId, long lineCount, CancellationToken cancellationToken)
@@ -152,7 +158,7 @@ namespace GitHub.Services.Results.Client
};
var createJobLogsMetadataEndpoint = new Uri(m_resultsServiceUrl, Constants.CreateJobLogsMetadata);
await CreateMetadata<JobLogsMetadataCreate>(createJobLogsMetadataEndpoint, cancellationToken, request, timestamp);
await SendRequest<JobLogsMetadataCreate>(createJobLogsMetadataEndpoint, cancellationToken, request, timestamp);
}
private async Task<HttpResponseMessage> UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken)
@@ -252,7 +258,7 @@ namespace GitHub.Services.Results.Client
await StepSummaryUploadCompleteAsync(planId, jobId, stepId, fileSize, cancellationToken);
}
// Handle file upload for step log
// Handle file upload for step log
public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken)
{
// Get the upload url
@@ -262,7 +268,7 @@ namespace GitHub.Services.Results.Client
throw new Exception("Failed to get step log upload url");
}
// Create the Append blob
// Create the Append blob
if (firstBlock)
{
await CreateAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken);
@@ -283,7 +289,7 @@ namespace GitHub.Services.Results.Client
}
}
// Handle file upload for job log
// Handle file upload for job log
public async Task UploadResultsJobLogAsync(string planId, string jobId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken)
{
// Get the upload url
@@ -293,7 +299,7 @@ namespace GitHub.Services.Results.Client
throw new Exception("Failed to get job log upload url");
}
// Create the Append blob
// Create the Append blob
if (firstBlock)
{
await CreateAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken);
@@ -314,9 +320,57 @@ namespace GitHub.Services.Results.Client
}
}
private Step ConvertTimelineRecordToStep(TimelineRecord r)
{
return new Step()
{
ExternalId = r.Id.ToString(),
Number = r.Order.GetValueOrDefault(),
Name = r.Name,
Status = ConvertStateToStatus(r.State.GetValueOrDefault()),
StartedAt = r.StartTime?.ToString(Constants.TimestampFormat),
CompletedAt = r.FinishTime?.ToString(Constants.TimestampFormat)
};
}
private Status ConvertStateToStatus(TimelineRecordState s)
{
switch (s)
{
case TimelineRecordState.Completed:
return Status.StatusCompleted;
case TimelineRecordState.Pending:
return Status.StatusPending;
case TimelineRecordState.InProgress:
return Status.StatusInProgress;
default:
return Status.StatusUnknown;
}
}
public async Task UpdateWorkflowStepsAsync(Guid planId, IEnumerable<TimelineRecord> records, CancellationToken cancellationToken)
{
var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat);
var stepRecords = records.Where(r => String.Equals(r.RecordType, "Task", StringComparison.Ordinal));
var stepUpdateRequests = stepRecords.GroupBy(r => r.ParentId).Select(sg => new StepsUpdateRequest()
{
WorkflowRunBackendId = planId.ToString(),
WorkflowJobRunBackendId = sg.Key.ToString(),
ChangeOrder = m_changeIdCounter++,
Steps = sg.Select(ConvertTimelineRecordToStep)
});
var stepUpdateEndpoint = new Uri(m_resultsServiceUrl, Constants.WorkflowStepsUpdate);
foreach (var request in stepUpdateRequests)
{
await SendRequest<StepsUpdateRequest>(stepUpdateEndpoint, cancellationToken, request, timestamp);
}
}
private MediaTypeFormatter m_formatter;
private Uri m_resultsServiceUrl;
private string m_token;
private int m_changeIdCounter;
}
// Constants specific to results
@@ -331,6 +385,8 @@ namespace GitHub.Services.Results.Client
public static readonly string CreateStepLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateStepLogsMetadata";
public static readonly string GetJobLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetJobLogsSignedBlobURL";
public static readonly string CreateJobLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateJobLogsMetadata";
public static readonly string ResultsProtoApiV1Endpoint = "twirp/github.actions.results.api.v1.WorkflowStepUpdateService/";
public static readonly string WorkflowStepsUpdate = ResultsProtoApiV1Endpoint + "WorkflowStepsUpdate";
public static readonly string AzureBlobSealedHeader = "x-ms-blob-sealed";
public static readonly string AzureBlobTypeHeader = "x-ms-blob-type";