From ac39c4bd0a281cff72961fc7f47126eec8fa3efc Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Mon, 8 Jan 2024 16:13:06 -0500 Subject: [PATCH] Use Azure SDK to upload files to Azure Blob (#3033) --- src/Misc/contentHash/dotnetRuntime/linux-arm | 2 +- .../contentHash/dotnetRuntime/linux-arm64 | 2 +- src/Misc/contentHash/dotnetRuntime/linux-x64 | 2 +- src/Misc/contentHash/dotnetRuntime/osx-arm64 | 2 +- src/Misc/contentHash/dotnetRuntime/osx-x64 | 2 +- src/Misc/contentHash/dotnetRuntime/win-arm64 | 2 +- src/Misc/contentHash/dotnetRuntime/win-x64 | 2 +- src/Misc/runnercoreassets | 9 +- src/Misc/runnerdotnetruntimeassets | 1 - src/Runner.Common/JobServerQueue.cs | 4 +- src/Runner.Common/ResultsServer.cs | 10 +- src/Sdk/Sdk.csproj | 1 + src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 209 +++++++++++++----- 13 files changed, 175 insertions(+), 73 deletions(-) diff --git a/src/Misc/contentHash/dotnetRuntime/linux-arm b/src/Misc/contentHash/dotnetRuntime/linux-arm index c750b23ed..9f55d62ef 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-arm +++ b/src/Misc/contentHash/dotnetRuntime/linux-arm @@ -1 +1 @@ -531b31914e525ecb12cc5526415bc70a112ebc818f877347af1a231011f539c5 \ No newline at end of file +54d95a44d118dba852395991224a6b9c1abe916858c87138656f80c619e85331 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/linux-arm64 b/src/Misc/contentHash/dotnetRuntime/linux-arm64 index 3380d9dcb..c03c98ade 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-arm64 +++ b/src/Misc/contentHash/dotnetRuntime/linux-arm64 @@ -1 +1 @@ -722dd5fa5ecc207fcccf67f6e502d689f2119d8117beff2041618fba17dc66a4 \ No newline at end of file +68015af17f06a824fa478e62ae7393766ce627fd5599ab916432a14656a19a52 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/linux-x64 b/src/Misc/contentHash/dotnetRuntime/linux-x64 index b2f1fc1a7..95a7155f7 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-x64 +++ b/src/Misc/contentHash/dotnetRuntime/linux-x64 @@ -1 +1 @@ -8ca75c76e15ab9dc7fe49a66c5c74e171e7fabd5d26546fda8931bd11bff30f9 \ No newline at end of file +a2628119ca419cb54e279103ffae7986cdbd0814d57c73ff0dc74c38be08b9ae \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/osx-arm64 b/src/Misc/contentHash/dotnetRuntime/osx-arm64 index 783fa8b55..d99ff5942 100644 --- a/src/Misc/contentHash/dotnetRuntime/osx-arm64 +++ b/src/Misc/contentHash/dotnetRuntime/osx-arm64 @@ -1 +1 @@ -70496eb1c99b39b3373b5088c95a35ebbaac1098e6c47c8aab94771f3ffbf501 \ No newline at end of file +de71ca09ead807e1a2ce9df0a5b23eb7690cb71fff51169a77e4c3992be53dda \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/osx-x64 b/src/Misc/contentHash/dotnetRuntime/osx-x64 index f59327329..085b329b2 100644 --- a/src/Misc/contentHash/dotnetRuntime/osx-x64 +++ b/src/Misc/contentHash/dotnetRuntime/osx-x64 @@ -1 +1 @@ -4f8d48727d535daabcaec814e0dafb271c10625366c78e7e022ca7477a73023f \ No newline at end of file +d009e05e6b26d614d65be736a15d1bd151932121c16a9ff1b986deadecc982b9 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/win-arm64 b/src/Misc/contentHash/dotnetRuntime/win-arm64 index d050cb89e..5c84f556e 100644 --- a/src/Misc/contentHash/dotnetRuntime/win-arm64 +++ b/src/Misc/contentHash/dotnetRuntime/win-arm64 @@ -1 +1 @@ -d54d7428f2b9200a0030365a6a4e174e30a1b29b922f8254dffb2924bd09549d \ No newline at end of file +f730db39c2305800b4653795360ba9c10c68f384a46b85d808f1f9f0ed3c42e4 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/win-x64 b/src/Misc/contentHash/dotnetRuntime/win-x64 index 881293ccb..6be8253b1 100644 --- a/src/Misc/contentHash/dotnetRuntime/win-x64 +++ b/src/Misc/contentHash/dotnetRuntime/win-x64 @@ -1 +1 @@ -eaa939c45307f46b7003902255b3a2a09287215d710984107667e03ac493eb26 \ No newline at end of file +a35b5722375490e9473cdcccb5e18b41eba3dbf4344fe31abc9821e21f18ea5a \ No newline at end of file diff --git a/src/Misc/runnercoreassets b/src/Misc/runnercoreassets index 34f873d5b..46354a5d9 100644 --- a/src/Misc/runnercoreassets +++ b/src/Misc/runnercoreassets @@ -6,6 +6,10 @@ darwin.svc.sh.template hashFiles/index.js installdependencies.sh macos-run-invoker.js +Azure.Core.dll +Azure.Storage.Blobs.dll +Azure.Storage.Common.dll +Microsoft.Bcl.AsyncInterfaces.dll Microsoft.IdentityModel.Logging.dll Microsoft.IdentityModel.Tokens.dll Minimatch.dll @@ -46,7 +50,10 @@ runsvc.sh Sdk.deps.json Sdk.dll Sdk.pdb +System.Diagnostics.DiagnosticSource.dll System.IdentityModel.Tokens.Jwt.dll +System.IO.Hashing.dll +System.Memory.Data.dll System.Net.Http.Formatting.dll System.Security.Cryptography.Pkcs.dll System.Security.Cryptography.ProtectedData.dll @@ -54,4 +61,4 @@ System.ServiceProcess.ServiceController.dll systemd.svc.sh.template update.cmd.template update.sh.template -YamlDotNet.dll \ No newline at end of file +YamlDotNet.dll diff --git a/src/Misc/runnerdotnetruntimeassets b/src/Misc/runnerdotnetruntimeassets index 3d9d1ea0a..32d947339 100644 --- a/src/Misc/runnerdotnetruntimeassets +++ b/src/Misc/runnerdotnetruntimeassets @@ -106,7 +106,6 @@ System.Data.DataSetExtensions.dll System.Data.dll System.Diagnostics.Contracts.dll System.Diagnostics.Debug.dll -System.Diagnostics.DiagnosticSource.dll System.Diagnostics.FileVersionInfo.dll System.Diagnostics.Process.dll System.Diagnostics.StackTrace.dll diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index e6a00f1c8..04450e011 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -134,8 +134,8 @@ namespace GitHub.Runner.Common { liveConsoleFeedUrl = feedStreamUrl; } - - _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken); + jobRequest.Variables.TryGetValue("system.github.results_upload_with_sdk", out VariableValue resultsUseSdkVariable); + _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken, StringUtil.ConvertToBoolean(resultsUseSdkVariable?.Value)); _resultsClientInitiated = true; } diff --git a/src/Runner.Common/ResultsServer.cs b/src/Runner.Common/ResultsServer.cs index ef97ebcfc..f3bf9910c 100644 --- a/src/Runner.Common/ResultsServer.cs +++ b/src/Runner.Common/ResultsServer.cs @@ -19,7 +19,7 @@ namespace GitHub.Runner.Common [ServiceLocator(Default = typeof(ResultServer))] public interface IResultsServer : IRunnerService, IAsyncDisposable { - void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token); + void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token, bool useSdk); Task AppendLiveConsoleFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList lines, long? startLine, CancellationToken cancellationToken); @@ -51,9 +51,9 @@ namespace GitHub.Runner.Common private String _liveConsoleFeedUrl; private string _token; - public void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token) + public void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token, bool useSdk) { - this._resultsClient = CreateHttpClient(uri, token); + this._resultsClient = CreateHttpClient(uri, token, useSdk); _token = token; if (!string.IsNullOrEmpty(liveConsoleFeedUrl)) @@ -63,7 +63,7 @@ namespace GitHub.Runner.Common } } - public ResultsHttpClient CreateHttpClient(Uri uri, string token) + public ResultsHttpClient CreateHttpClient(Uri uri, string token, bool useSdk) { // Using default 100 timeout RawClientHttpRequestSettings settings = VssUtil.GetHttpRequestSettings(null); @@ -80,7 +80,7 @@ namespace GitHub.Runner.Common var pipeline = HttpClientFactory.CreatePipeline(httpMessageHandler, delegatingHandlers); - return new ResultsHttpClient(uri, pipeline, token, disposeHandler: true); + return new ResultsHttpClient(uri, pipeline, token, disposeHandler: true, useSdk: useSdk); } public Task CreateResultsStepSummaryAsync(string planId, string jobId, Guid stepId, string file, diff --git a/src/Sdk/Sdk.csproj b/src/Sdk/Sdk.csproj index dbd96f336..ff1cb85a4 100644 --- a/src/Sdk/Sdk.csproj +++ b/src/Sdk/Sdk.csproj @@ -13,6 +13,7 @@ + diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 3bdc8cc02..9a7eb990b 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Net.Http; @@ -8,8 +7,11 @@ using System.Net.Http.Headers; using System.Threading; using System.Threading.Tasks; using System.Net.Http.Formatting; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; using GitHub.DistributedTask.WebApi; -using GitHub.Services.Common; using GitHub.Services.Results.Contracts; using Sdk.WebApi.WebApi; @@ -21,13 +23,15 @@ namespace GitHub.Services.Results.Client Uri baseUrl, HttpMessageHandler pipeline, string token, - bool disposeHandler) + bool disposeHandler, + bool useSdk) : base(baseUrl, pipeline, disposeHandler) { m_token = token; m_resultsServiceUrl = baseUrl; m_formatter = new JsonMediaTypeFormatter(); m_changeIdCounter = 1; + m_useSdk = useSdk; } // Get Sas URL calls @@ -91,7 +95,6 @@ namespace GitHub.Services.Results.Client } // Create metadata calls - private async Task SendRequest(Uri uri, CancellationToken cancellationToken, R request, string timestamp) { using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, uri)) @@ -161,73 +164,164 @@ namespace GitHub.Services.Results.Client await SendRequest(createJobLogsMetadataEndpoint, cancellationToken, request, timestamp); } - private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) + private (Uri path, string sas) ParseSasToken(string url) { - // Upload the file to the url - var request = new HttpRequestMessage(HttpMethod.Put, url) + if (String.IsNullOrEmpty(url)) { - Content = new StreamContent(file) - }; - - if (blobStorageType == BlobStorageTypes.AzureBlobStorage) - { - request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureBlockBlob); + throw new Exception($"SAS url is empty"); } - using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) + var blobUri = new UriBuilder(url); + var sasUrl = blobUri.Query.Substring(1); //remove starting "?" + blobUri.Query = null; // remove query params + return (blobUri.Uri, sasUrl); + } + + private BlobClient GetBlobClient(string url) + { + var blobUri = ParseSasToken(url); + + var opts = new BlobClientOptions { - if (!response.IsSuccessStatusCode) + Retry = { - throw new Exception($"Failed to upload file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}"); + MaxRetries = Constants.DefaultBlobUploadRetries, + NetworkTimeout = TimeSpan.FromSeconds(Constants.DefaultNetworkTimeoutInSeconds) + } + }; + + return new BlobClient(blobUri.path, new AzureSasCredential(blobUri.sas), opts); + } + + private AppendBlobClient GetAppendBlobClient(string url) + { + var blobUri = ParseSasToken(url); + + var opts = new BlobClientOptions + { + Retry = + { + MaxRetries = Constants.DefaultBlobUploadRetries, + NetworkTimeout = TimeSpan.FromSeconds(Constants.DefaultNetworkTimeoutInSeconds) + } + }; + + return new AppendBlobClient(blobUri.path, new AzureSasCredential(blobUri.sas), opts); + } + + private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) + { + if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage) + { + var blobClient = GetBlobClient(url); + try + { + await blobClient.UploadAsync(file, cancellationToken); + } + catch (RequestFailedException e) + { + throw new Exception($"Failed to upload block to Azure blob: {e.Message}"); + } + } + else + { + // Upload the file to the url + var request = new HttpRequestMessage(HttpMethod.Put, url) + { + Content = new StreamContent(file) + }; + + if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + { + request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureBlockBlob); + } + + using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) + { + if (!response.IsSuccessStatusCode) + { + throw new Exception($"Failed to upload file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}"); + } } - return response; } } - private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken) + private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken) { - var request = new HttpRequestMessage(HttpMethod.Put, url) + if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage) { - Content = new StringContent("") - }; - if (blobStorageType == BlobStorageTypes.AzureBlobStorage) - { - request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureAppendBlob); - request.Content.Headers.Add("Content-Length", "0"); - } - - using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) - { - if (!response.IsSuccessStatusCode) + var appendBlobClient = GetAppendBlobClient(url); + try { - throw new Exception($"Failed to create append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}"); + await appendBlobClient.CreateAsync(cancellationToken: cancellationToken); + } + catch (RequestFailedException e) + { + throw new Exception($"Failed to create append blob in Azure blob: {e.Message}"); + } + } + else + { + var request = new HttpRequestMessage(HttpMethod.Put, url) + { + Content = new StringContent("") + }; + if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + { + request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureAppendBlob); + request.Content.Headers.Add("Content-Length", "0"); + } + + using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) + { + if (!response.IsSuccessStatusCode) + { + throw new Exception($"Failed to create append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}"); + } } - return response; } } - private async Task UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken) + private async Task UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken) { - var comp = finalize ? "&comp=appendblock&seal=true" : "&comp=appendblock"; - // Upload the file to the url - var request = new HttpRequestMessage(HttpMethod.Put, url + comp) + if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage) { - Content = new StreamContent(file) - }; - - if (blobStorageType == BlobStorageTypes.AzureBlobStorage) - { - request.Content.Headers.Add("Content-Length", fileSize.ToString()); - request.Content.Headers.Add(Constants.AzureBlobSealedHeader, finalize.ToString()); - } - - using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) - { - if (!response.IsSuccessStatusCode) + var appendBlobClient = GetAppendBlobClient(url); + try { - throw new Exception($"Failed to upload append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}, object: {response}, fileSize: {fileSize}"); + await appendBlobClient.AppendBlockAsync(file, cancellationToken: cancellationToken); + if (finalize) + { + await appendBlobClient.SealAsync(cancellationToken: cancellationToken); + } + } + catch (RequestFailedException e) + { + throw new Exception($"Failed to upload append block in Azure blob: {e.Message}"); + } + } + else + { + var comp = finalize ? "&comp=appendblock&seal=true" : "&comp=appendblock"; + // Upload the file to the url + var request = new HttpRequestMessage(HttpMethod.Put, url + comp) + { + Content = new StreamContent(file) + }; + + if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + { + request.Content.Headers.Add("Content-Length", fileSize.ToString()); + request.Content.Headers.Add(Constants.AzureBlobSealedHeader, finalize.ToString()); + } + + using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) + { + if (!response.IsSuccessStatusCode) + { + throw new Exception($"Failed to upload append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}, object: {response}, fileSize: {fileSize}"); + } } - return response; } } @@ -251,23 +345,22 @@ namespace GitHub.Services.Results.Client // Upload the file using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) { - var response = await UploadBlockFileAsync(uploadUrlResponse.SummaryUrl, uploadUrlResponse.BlobStorageType, fileStream, cancellationToken); + await UploadBlockFileAsync(uploadUrlResponse.SummaryUrl, uploadUrlResponse.BlobStorageType, fileStream, cancellationToken); } // Send step summary upload complete message await StepSummaryUploadCompleteAsync(planId, jobId, stepId, fileSize, cancellationToken); } - private async Task UploadLogFile(string file, bool finalize, bool firstBlock, string sasUrl, string blobStorageType, + private async Task UploadLogFile(string file, bool finalize, bool firstBlock, string sasUrl, string blobStorageType, CancellationToken cancellationToken) { - HttpResponseMessage response; if (firstBlock && finalize) { // This is the one and only block, just use a block blob using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) { - response = await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken); + await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken); } } else @@ -283,11 +376,9 @@ namespace GitHub.Services.Results.Client var fileSize = new FileInfo(file).Length; using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) { - response = await UploadAppendFileAsync(sasUrl, blobStorageType, fileStream, finalize, fileSize, cancellationToken); + await UploadAppendFileAsync(sasUrl, blobStorageType, fileStream, finalize, fileSize, cancellationToken); } } - - return response; } // Handle file upload for step log @@ -405,6 +496,7 @@ namespace GitHub.Services.Results.Client private Uri m_resultsServiceUrl; private string m_token; private int m_changeIdCounter; + private bool m_useSdk; } // Constants specific to results @@ -422,6 +514,9 @@ namespace GitHub.Services.Results.Client public static readonly string ResultsProtoApiV1Endpoint = "twirp/github.actions.results.api.v1.WorkflowStepUpdateService/"; public static readonly string WorkflowStepsUpdate = ResultsProtoApiV1Endpoint + "WorkflowStepsUpdate"; + public static readonly int DefaultNetworkTimeoutInSeconds = 30; + public static readonly int DefaultBlobUploadRetries = 3; + public static readonly string AzureBlobSealedHeader = "x-ms-blob-sealed"; public static readonly string AzureBlobTypeHeader = "x-ms-blob-type"; public static readonly string AzureBlockBlob = "BlockBlob";