actions/runner
1
0
Fork 0
mirror of https://github.com/actions/runner.git synced 2025-12-12 15:13:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use Azure SDK to upload files to Azure Blob (#3033)

Browse Source
This commit is contained in:
Yang Cao
2024-01-08 16:13:06 -05:00
committed by GitHub
parent 3f3d9b0d99
commit ac39c4bd0a
13 changed files with 175 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/Misc/contentHash/dotnetRuntime/linux-arm
View File

@@ -1 +1 @@
531b31914e525ecb12cc5526415bc70a112ebc818f877347af1a231011f539c5
54d95a44d118dba852395991224a6b9c1abe916858c87138656f80c619e85331

2
src/Misc/contentHash/dotnetRuntime/linux-arm64
View File

@@ -1 +1 @@
722dd5fa5ecc207fcccf67f6e502d689f2119d8117beff2041618fba17dc66a4
68015af17f06a824fa478e62ae7393766ce627fd5599ab916432a14656a19a52

2
src/Misc/contentHash/dotnetRuntime/linux-x64
View File

@@ -1 +1 @@
8ca75c76e15ab9dc7fe49a66c5c74e171e7fabd5d26546fda8931bd11bff30f9
a2628119ca419cb54e279103ffae7986cdbd0814d57c73ff0dc74c38be08b9ae

2
src/Misc/contentHash/dotnetRuntime/osx-arm64
View File

@@ -1 +1 @@
70496eb1c99b39b3373b5088c95a35ebbaac1098e6c47c8aab94771f3ffbf501
de71ca09ead807e1a2ce9df0a5b23eb7690cb71fff51169a77e4c3992be53dda

2
src/Misc/contentHash/dotnetRuntime/osx-x64
View File

@@ -1 +1 @@
4f8d48727d535daabcaec814e0dafb271c10625366c78e7e022ca7477a73023f
d009e05e6b26d614d65be736a15d1bd151932121c16a9ff1b986deadecc982b9

2
src/Misc/contentHash/dotnetRuntime/win-arm64
View File

@@ -1 +1 @@
d54d7428f2b9200a0030365a6a4e174e30a1b29b922f8254dffb2924bd09549d
f730db39c2305800b4653795360ba9c10c68f384a46b85d808f1f9f0ed3c42e4

2
src/Misc/contentHash/dotnetRuntime/win-x64
View File

@@ -1 +1 @@
eaa939c45307f46b7003902255b3a2a09287215d710984107667e03ac493eb26
a35b5722375490e9473cdcccb5e18b41eba3dbf4344fe31abc9821e21f18ea5a

9
src/Misc/runnercoreassets
View File

@@ -6,6 +6,10 @@ darwin.svc.sh.template
hashFiles/index.js
installdependencies.sh
macos-run-invoker.js
Azure.Core.dll
Azure.Storage.Blobs.dll
Azure.Storage.Common.dll
Microsoft.Bcl.AsyncInterfaces.dll
Microsoft.IdentityModel.Logging.dll
Microsoft.IdentityModel.Tokens.dll
Minimatch.dll
@@ -46,7 +50,10 @@ runsvc.sh
Sdk.deps.json
Sdk.dll
Sdk.pdb
System.Diagnostics.DiagnosticSource.dll
System.IdentityModel.Tokens.Jwt.dll
System.IO.Hashing.dll
System.Memory.Data.dll
System.Net.Http.Formatting.dll
System.Security.Cryptography.Pkcs.dll
System.Security.Cryptography.ProtectedData.dll
@@ -54,4 +61,4 @@ System.ServiceProcess.ServiceController.dll
systemd.svc.sh.template
update.cmd.template
update.sh.template
YamlDotNet.dll
YamlDotNet.dll

1
src/Misc/runnerdotnetruntimeassets
View File

@@ -106,7 +106,6 @@ System.Data.DataSetExtensions.dll
System.Data.dll
System.Diagnostics.Contracts.dll
System.Diagnostics.Debug.dll
System.Diagnostics.DiagnosticSource.dll
System.Diagnostics.FileVersionInfo.dll
System.Diagnostics.Process.dll
System.Diagnostics.StackTrace.dll

4
src/Runner.Common/JobServerQueue.cs
View File

@@ -134,8 +134,8 @@ namespace GitHub.Runner.Common
{
liveConsoleFeedUrl = feedStreamUrl;
}
_resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken);
jobRequest.Variables.TryGetValue("system.github.results_upload_with_sdk", out VariableValue resultsUseSdkVariable);
_resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken, StringUtil.ConvertToBoolean(resultsUseSdkVariable?.Value));
_resultsClientInitiated = true;
}

10
src/Runner.Common/ResultsServer.cs
View File

@@ -19,7 +19,7 @@ namespace GitHub.Runner.Common
[ServiceLocator(Default = typeof(ResultServer))]
public interface IResultsServer : IRunnerService, IAsyncDisposable
{
void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token);
void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token, bool useSdk);
Task<bool> AppendLiveConsoleFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, long? startLine, CancellationToken cancellationToken);
@@ -51,9 +51,9 @@ namespace GitHub.Runner.Common
private String _liveConsoleFeedUrl;
private string _token;
public void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token)
public void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token, bool useSdk)
{
this._resultsClient = CreateHttpClient(uri, token);
this._resultsClient = CreateHttpClient(uri, token, useSdk);
_token = token;
if (!string.IsNullOrEmpty(liveConsoleFeedUrl))
@@ -63,7 +63,7 @@ namespace GitHub.Runner.Common
}
}
public ResultsHttpClient CreateHttpClient(Uri uri, string token)
public ResultsHttpClient CreateHttpClient(Uri uri, string token, bool useSdk)
{
// Using default 100 timeout
RawClientHttpRequestSettings settings = VssUtil.GetHttpRequestSettings(null);
@@ -80,7 +80,7 @@ namespace GitHub.Runner.Common
var pipeline = HttpClientFactory.CreatePipeline(httpMessageHandler, delegatingHandlers);
return new ResultsHttpClient(uri, pipeline, token, disposeHandler: true);
return new ResultsHttpClient(uri, pipeline, token, disposeHandler: true, useSdk: useSdk);
}
public Task CreateResultsStepSummaryAsync(string planId, string jobId, Guid stepId, string file,

1
src/Sdk/Sdk.csproj
View File

@@ -13,6 +13,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.Storage.Blobs" Version="12.19.1" />
<PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Microsoft.AspNet.WebApi.Client" Version="5.2.9" />

209
src/Sdk/WebApi/WebApi/ResultsHttpClient.cs
View File

@@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http;
@@ -8,8 +7,11 @@ using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using System.Net.Http.Formatting;
using Azure;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Azure.Storage.Blobs.Specialized;
using GitHub.DistributedTask.WebApi;
using GitHub.Services.Common;
using GitHub.Services.Results.Contracts;
using Sdk.WebApi.WebApi;
@@ -21,13 +23,15 @@ namespace GitHub.Services.Results.Client
Uri baseUrl,
HttpMessageHandler pipeline,
string token,
bool disposeHandler)
bool disposeHandler,
bool useSdk)
: base(baseUrl, pipeline, disposeHandler)
{
m_token = token;
m_resultsServiceUrl = baseUrl;
m_formatter = new JsonMediaTypeFormatter();
m_changeIdCounter = 1;
m_useSdk = useSdk;
}
// Get Sas URL calls
@@ -91,7 +95,6 @@ namespace GitHub.Services.Results.Client
}
// Create metadata calls
private async Task SendRequest<R>(Uri uri, CancellationToken cancellationToken, R request, string timestamp)
{
using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, uri))
@@ -161,73 +164,164 @@ namespace GitHub.Services.Results.Client
await SendRequest<JobLogsMetadataCreate>(createJobLogsMetadataEndpoint, cancellationToken, request, timestamp);
}
private async Task<HttpResponseMessage> UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken)
private (Uri path, string sas) ParseSasToken(string url)
{
// Upload the file to the url
var request = new HttpRequestMessage(HttpMethod.Put, url)
if (String.IsNullOrEmpty(url))
{
Content = new StreamContent(file)
};
if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureBlockBlob);
throw new Exception($"SAS url is empty");
}
using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
var blobUri = new UriBuilder(url);
var sasUrl = blobUri.Query.Substring(1); //remove starting "?"
blobUri.Query = null; // remove query params
return (blobUri.Uri, sasUrl);
}
private BlobClient GetBlobClient(string url)
{
var blobUri = ParseSasToken(url);
var opts = new BlobClientOptions
{
if (!response.IsSuccessStatusCode)
Retry =
{
throw new Exception($"Failed to upload file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}");
MaxRetries = Constants.DefaultBlobUploadRetries,
NetworkTimeout = TimeSpan.FromSeconds(Constants.DefaultNetworkTimeoutInSeconds)
}
};
return new BlobClient(blobUri.path, new AzureSasCredential(blobUri.sas), opts);
}
private AppendBlobClient GetAppendBlobClient(string url)
{
var blobUri = ParseSasToken(url);
var opts = new BlobClientOptions
{
Retry =
{
MaxRetries = Constants.DefaultBlobUploadRetries,
NetworkTimeout = TimeSpan.FromSeconds(Constants.DefaultNetworkTimeoutInSeconds)
}
};
return new AppendBlobClient(blobUri.path, new AzureSasCredential(blobUri.sas), opts);
}
private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken)
{
if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
var blobClient = GetBlobClient(url);
try
{
await blobClient.UploadAsync(file, cancellationToken);
}
catch (RequestFailedException e)
{
throw new Exception($"Failed to upload block to Azure blob: {e.Message}");
}
}
else
{
// Upload the file to the url
var request = new HttpRequestMessage(HttpMethod.Put, url)
{
Content = new StreamContent(file)
};
if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureBlockBlob);
}
using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
{
if (!response.IsSuccessStatusCode)
{
throw new Exception($"Failed to upload file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}");
}
}
return response;
}
}
private async Task<HttpResponseMessage> CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken)
private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken)
{
var request = new HttpRequestMessage(HttpMethod.Put, url)
if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
Content = new StringContent("")
};
if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureAppendBlob);
request.Content.Headers.Add("Content-Length", "0");
}
using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
{
if (!response.IsSuccessStatusCode)
var appendBlobClient = GetAppendBlobClient(url);
try
{
throw new Exception($"Failed to create append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}");
await appendBlobClient.CreateAsync(cancellationToken: cancellationToken);
}
catch (RequestFailedException e)
{
throw new Exception($"Failed to create append blob in Azure blob: {e.Message}");
}
}
else
{
var request = new HttpRequestMessage(HttpMethod.Put, url)
{
Content = new StringContent("")
};
if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureAppendBlob);
request.Content.Headers.Add("Content-Length", "0");
}
using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
{
if (!response.IsSuccessStatusCode)
{
throw new Exception($"Failed to create append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}");
}
}
return response;
}
}
private async Task<HttpResponseMessage> UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken)
private async Task UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken)
{
var comp = finalize ? "&comp=appendblock&seal=true" : "&comp=appendblock";
// Upload the file to the url
var request = new HttpRequestMessage(HttpMethod.Put, url + comp)
if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
Content = new StreamContent(file)
};
if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
request.Content.Headers.Add("Content-Length", fileSize.ToString());
request.Content.Headers.Add(Constants.AzureBlobSealedHeader, finalize.ToString());
}
using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
{
if (!response.IsSuccessStatusCode)
var appendBlobClient = GetAppendBlobClient(url);
try
{
throw new Exception($"Failed to upload append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}, object: {response}, fileSize: {fileSize}");
await appendBlobClient.AppendBlockAsync(file, cancellationToken: cancellationToken);
if (finalize)
{
await appendBlobClient.SealAsync(cancellationToken: cancellationToken);
}
}
catch (RequestFailedException e)
{
throw new Exception($"Failed to upload append block in Azure blob: {e.Message}");
}
}
else
{
var comp = finalize ? "&comp=appendblock&seal=true" : "&comp=appendblock";
// Upload the file to the url
var request = new HttpRequestMessage(HttpMethod.Put, url + comp)
{
Content = new StreamContent(file)
};
if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
request.Content.Headers.Add("Content-Length", fileSize.ToString());
request.Content.Headers.Add(Constants.AzureBlobSealedHeader, finalize.ToString());
}
using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
{
if (!response.IsSuccessStatusCode)
{
throw new Exception($"Failed to upload append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}, object: {response}, fileSize: {fileSize}");
}
}
return response;
}
}
@@ -251,23 +345,22 @@ namespace GitHub.Services.Results.Client
// Upload the file
using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
{
var response = await UploadBlockFileAsync(uploadUrlResponse.SummaryUrl, uploadUrlResponse.BlobStorageType, fileStream, cancellationToken);
await UploadBlockFileAsync(uploadUrlResponse.SummaryUrl, uploadUrlResponse.BlobStorageType, fileStream, cancellationToken);
}
// Send step summary upload complete message
await StepSummaryUploadCompleteAsync(planId, jobId, stepId, fileSize, cancellationToken);
}
private async Task<HttpResponseMessage> UploadLogFile(string file, bool finalize, bool firstBlock, string sasUrl, string blobStorageType,
private async Task UploadLogFile(string file, bool finalize, bool firstBlock, string sasUrl, string blobStorageType,
CancellationToken cancellationToken)
{
HttpResponseMessage response;
if (firstBlock && finalize)
{
// This is the one and only block, just use a block blob
using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
{
response = await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken);
await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken);
}
}
else
@@ -283,11 +376,9 @@ namespace GitHub.Services.Results.Client
var fileSize = new FileInfo(file).Length;
using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
{
response = await UploadAppendFileAsync(sasUrl, blobStorageType, fileStream, finalize, fileSize, cancellationToken);
await UploadAppendFileAsync(sasUrl, blobStorageType, fileStream, finalize, fileSize, cancellationToken);
}
}
return response;
}
// Handle file upload for step log
@@ -405,6 +496,7 @@ namespace GitHub.Services.Results.Client
private Uri m_resultsServiceUrl;
private string m_token;
private int m_changeIdCounter;
private bool m_useSdk;
}
// Constants specific to results
@@ -422,6 +514,9 @@ namespace GitHub.Services.Results.Client
public static readonly string ResultsProtoApiV1Endpoint = "twirp/github.actions.results.api.v1.WorkflowStepUpdateService/";
public static readonly string WorkflowStepsUpdate = ResultsProtoApiV1Endpoint + "WorkflowStepsUpdate";
public static readonly int DefaultNetworkTimeoutInSeconds = 30;
public static readonly int DefaultBlobUploadRetries = 3;
public static readonly string AzureBlobSealedHeader = "x-ms-blob-sealed";
public static readonly string AzureBlobTypeHeader = "x-ms-blob-type";
public static readonly string AzureBlockBlob = "BlockBlob";