From de29a39d141110ebbddaec67e72a30d6e81e15ff Mon Sep 17 00:00:00 2001 From: Julio Barba Date: Mon, 25 Nov 2019 13:30:44 -0500 Subject: [PATCH] Support downloading/publishing artifacts from Pipelines endpoint (#188) * Support downloading/publishing artifacts from Pipelines endpoint * Remove `Path` from everywhere * Remove unused JobId argument * PR feedback * More PR feedback --- .../Artifact/DownloadArtifact.cs | 57 +++- .../Artifact/PipelinesServer.cs | 60 ++++ .../Artifact/PublishArtifact.cs | 46 ++- .../Contracts/ActionsStorageArtifact.cs | 44 +++ src/Sdk/PipelinesWebApi/Contracts/Artifact.cs | 46 +++ .../Contracts/ArtifactBaseJsonConverter.cs | 85 ++++++ .../Contracts/ArtifactJsonConverter.cs | 29 ++ .../PipelinesWebApi/Contracts/ArtifactType.cs | 14 + .../ArtifactTypeEnumJsonConverter.cs | 25 ++ .../CreateActionsStorageArtifactParameters.cs | 33 +++ .../Contracts/CreateArtifactParameters.cs | 35 +++ .../CreateArtifactParametersJsonConverter.cs | 29 ++ .../Contracts/GetArtifactExpandOptions.cs | 16 + src/Sdk/PipelinesWebApi/FlagsEnum.cs | 105 +++++++ .../Generated/PipelinesHttpClientBase.cs | 273 ++++++++++++++++++ .../KnownFlagsEnumTypeConverter.cs | 44 +++ .../PipelinesWebApi/PipelinesHttpClient.cs | 36 +++ .../PipelinesWebApi/PipelinesResourceIds.cs | 74 +++++ src/Sdk/PipelinesWebApi/UnknownEnum.cs | 45 +++ .../UnknownEnumJsonConverter.cs | 53 ++++ .../Resources/PipelinesWebApiResources.g.cs | 26 ++ .../WebApi/Contracts/SignedUrl/SignedUrl.cs | 18 ++ 22 files changed, 1174 insertions(+), 19 deletions(-) create mode 100644 src/Runner.Plugins/Artifact/PipelinesServer.cs create mode 100644 src/Sdk/PipelinesWebApi/Contracts/ActionsStorageArtifact.cs create mode 100644 src/Sdk/PipelinesWebApi/Contracts/Artifact.cs create mode 100644 src/Sdk/PipelinesWebApi/Contracts/ArtifactBaseJsonConverter.cs create mode 100644 src/Sdk/PipelinesWebApi/Contracts/ArtifactJsonConverter.cs create mode 100644 src/Sdk/PipelinesWebApi/Contracts/ArtifactType.cs create mode 100644 src/Sdk/PipelinesWebApi/Contracts/ArtifactTypeEnumJsonConverter.cs create mode 100644 src/Sdk/PipelinesWebApi/Contracts/CreateActionsStorageArtifactParameters.cs create mode 100644 src/Sdk/PipelinesWebApi/Contracts/CreateArtifactParameters.cs create mode 100644 src/Sdk/PipelinesWebApi/Contracts/CreateArtifactParametersJsonConverter.cs create mode 100644 src/Sdk/PipelinesWebApi/Contracts/GetArtifactExpandOptions.cs create mode 100644 src/Sdk/PipelinesWebApi/FlagsEnum.cs create mode 100644 src/Sdk/PipelinesWebApi/Generated/PipelinesHttpClientBase.cs create mode 100644 src/Sdk/PipelinesWebApi/KnownFlagsEnumTypeConverter.cs create mode 100644 src/Sdk/PipelinesWebApi/PipelinesHttpClient.cs create mode 100644 src/Sdk/PipelinesWebApi/PipelinesResourceIds.cs create mode 100644 src/Sdk/PipelinesWebApi/UnknownEnum.cs create mode 100644 src/Sdk/PipelinesWebApi/UnknownEnumJsonConverter.cs create mode 100644 src/Sdk/Resources/PipelinesWebApiResources.g.cs create mode 100644 src/Sdk/WebApi/WebApi/Contracts/SignedUrl/SignedUrl.cs diff --git a/src/Runner.Plugins/Artifact/DownloadArtifact.cs b/src/Runner.Plugins/Artifact/DownloadArtifact.cs index 4ea75a355..593b9fb49 100644 --- a/src/Runner.Plugins/Artifact/DownloadArtifact.cs +++ b/src/Runner.Plugins/Artifact/DownloadArtifact.cs @@ -50,29 +50,62 @@ namespace GitHub.Runner.Plugins.Artifact throw new ArgumentException($"Run Id is not an Int32: {buildIdStr}"); } - context.Output($"Download artifact '{artifactName}' to: '{targetPath}'"); + // Determine whether to call Pipelines or Build endpoint to publish artifact based on variable setting + string usePipelinesArtifactEndpointVar = context.Variables.GetValueOrDefault("Runner.UseActionsArtifactsApis")?.Value; + bool.TryParse(usePipelinesArtifactEndpointVar, out bool usePipelinesArtifactEndpoint); + string containerPath; + long containerId; - BuildServer buildHelper = new BuildServer(context.VssConnection); - BuildArtifact buildArtifact = await buildHelper.GetArtifact(projectId, buildId, artifactName, token); + context.Output($"Downloading artifact '{artifactName}' to: '{targetPath}'"); - if (string.Equals(buildArtifact.Resource.Type, "Container", StringComparison.OrdinalIgnoreCase)) + if (usePipelinesArtifactEndpoint) { - string containerUrl = buildArtifact.Resource.Data; - string[] parts = containerUrl.Split(new[] { '/' }, 3); - if (parts.Length < 3 || !long.TryParse(parts[1], out long containerId)) + context.Debug("Downloading artifact using v2 endpoint"); + + // Definition ID is a dummy value only used by HTTP client routing purposes + int definitionId = 1; + + var pipelinesHelper = new PipelinesServer(context.VssConnection); + + var actionsStorageArtifact = await pipelinesHelper.GetActionsStorageArtifact(definitionId, buildId, artifactName, token); + + if (actionsStorageArtifact == null) { - throw new ArgumentOutOfRangeException($"Invalid container url '{containerUrl}' for artifact '{buildArtifact.Name}'"); + throw new Exception($"The actions storage artifact for '{artifactName}' could not be found, or is no longer available"); } - string containerPath = parts[2]; - FileContainerServer fileContainerServer = new FileContainerServer(context.VssConnection, projectId, containerId, containerPath); - await fileContainerServer.DownloadFromContainerAsync(context, targetPath, token); + containerPath = actionsStorageArtifact.Name; // In actions storage artifacts, name equals the path + containerId = actionsStorageArtifact.ContainerId; } else { - throw new NotSupportedException($"Invalid artifact type: {buildArtifact.Resource.Type}"); + context.Debug("Downloading artifact using v1 endpoint"); + + BuildServer buildHelper = new BuildServer(context.VssConnection); + BuildArtifact buildArtifact = await buildHelper.GetArtifact(projectId, buildId, artifactName, token); + + if (string.Equals(buildArtifact.Resource.Type, "Container", StringComparison.OrdinalIgnoreCase) || + // Artifact was published by Pipelines endpoint, check new type here to handle rollback scenario + string.Equals(buildArtifact.Resource.Type, "Actions_Storage", StringComparison.OrdinalIgnoreCase)) + { + string containerUrl = buildArtifact.Resource.Data; + string[] parts = containerUrl.Split(new[] { '/' }, 3); + if (parts.Length < 3 || !long.TryParse(parts[1], out containerId)) + { + throw new ArgumentOutOfRangeException($"Invalid container url '{containerUrl}' for artifact '{buildArtifact.Name}'"); + } + + containerPath = parts[2]; + } + else + { + throw new NotSupportedException($"Invalid artifact type: {buildArtifact.Resource.Type}"); + } } + FileContainerServer fileContainerServer = new FileContainerServer(context.VssConnection, projectId, containerId, containerPath); + await fileContainerServer.DownloadFromContainerAsync(context, targetPath, token); + context.Output("Artifact download finished."); } } diff --git a/src/Runner.Plugins/Artifact/PipelinesServer.cs b/src/Runner.Plugins/Artifact/PipelinesServer.cs new file mode 100644 index 000000000..6f7124045 --- /dev/null +++ b/src/Runner.Plugins/Artifact/PipelinesServer.cs @@ -0,0 +1,60 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using GitHub.Actions.Pipelines.WebApi; +using GitHub.Services.WebApi; +using GitHub.Runner.Sdk; +using Pipelines = GitHub.Actions.Pipelines.WebApi; + +namespace GitHub.Runner.Plugins.Artifact +{ + // A client wrapper interacting with Pipelines's Artifact API + public class PipelinesServer + { + private readonly PipelinesHttpClient _pipelinesHttpClient; + + public PipelinesServer(VssConnection connection) + { + ArgUtil.NotNull(connection, nameof(connection)); + _pipelinesHttpClient = connection.GetClient(); + } + + // Associate the specified Actions Storage artifact with a pipeline + public async Task AssociateActionsStorageArtifactAsync( + int pipelineId, + int runId, + long containerId, + string name, + long size, + CancellationToken cancellationToken = default(CancellationToken)) + { + CreateArtifactParameters parameters = new CreateActionsStorageArtifactParameters() + { + Name = name, + ContainerId = containerId, + Size = size + }; + + return await _pipelinesHttpClient.CreateArtifactAsync( + parameters, + pipelineId, + runId, + cancellationToken: cancellationToken) as Pipelines.ActionsStorageArtifact; + } + + // Get named Actions Storage artifact for a pipeline + public async Task GetActionsStorageArtifact( + int pipelineId, + int runId, + string name, + CancellationToken cancellationToken) + { + return await _pipelinesHttpClient.GetArtifactAsync( + pipelineId, + runId, + name, + cancellationToken: cancellationToken) as Pipelines.ActionsStorageArtifact; + } + } +} diff --git a/src/Runner.Plugins/Artifact/PublishArtifact.cs b/src/Runner.Plugins/Artifact/PublishArtifact.cs index 3975c1e91..f72391238 100644 --- a/src/Runner.Plugins/Artifact/PublishArtifact.cs +++ b/src/Runner.Plugins/Artifact/PublishArtifact.cs @@ -68,27 +68,59 @@ namespace GitHub.Runner.Plugins.Artifact string containerIdStr = context.Variables.GetValueOrDefault(BuildVariables.ContainerId)?.Value ?? string.Empty; if (!long.TryParse(containerIdStr, out long containerId)) { - throw new ArgumentException($"Container Id is not a Int64: {containerIdStr}"); + throw new ArgumentException($"Container Id is not an Int64: {containerIdStr}"); } context.Output($"Uploading artifact '{artifactName}' from '{fullPath}' for run #{buildId}"); FileContainerServer fileContainerHelper = new FileContainerServer(context.VssConnection, projectId, containerId, artifactName); var propertiesDictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); + + long size = 0; + try { - long size = await fileContainerHelper.CopyToContainerAsync(context, fullPath, token); + size = await fileContainerHelper.CopyToContainerAsync(context, fullPath, token); + propertiesDictionary.Add("artifactsize", size.ToString()); + context.Output($"Uploaded '{size}' bytes from '{fullPath}' to server"); } // if any of the results were successful, make sure to attach them to the build finally { - string fileContainerFullPath = StringUtil.Format($"#/{containerId}/{artifactName}"); - BuildServer buildHelper = new BuildServer(context.VssConnection); - string jobId = context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId).Value ?? string.Empty; - var artifact = await buildHelper.AssociateArtifact(projectId, buildId, jobId, artifactName, ArtifactResourceTypes.Container, fileContainerFullPath, propertiesDictionary, token); - context.Output($"Associated artifact {artifactName} ({artifact.Id}) with run #{buildId}"); + // Determine whether to call Pipelines or Build endpoint to publish artifact based on variable setting + string usePipelinesArtifactEndpointVar = context.Variables.GetValueOrDefault("Runner.UseActionsArtifactsApis")?.Value; + bool.TryParse(usePipelinesArtifactEndpointVar, out bool usePipelinesArtifactEndpoint); + + if (usePipelinesArtifactEndpoint) + { + // Definition ID is a dummy value only used by HTTP client routing purposes + int definitionId = 1; + + PipelinesServer pipelinesHelper = new PipelinesServer(context.VssConnection); + + var artifact = await pipelinesHelper.AssociateActionsStorageArtifactAsync( + definitionId, + buildId, + containerId, + artifactName, + size, + token); + + context.Output($"Associated artifact {artifactName} ({artifact.ContainerId}) with run #{buildId}"); + context.Debug($"Associated artifact using v2 endpoint"); + } + else + { + string fileContainerFullPath = StringUtil.Format($"#/{containerId}/{artifactName}"); + BuildServer buildHelper = new BuildServer(context.VssConnection); + string jobId = context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId).Value ?? string.Empty; + var artifact = await buildHelper.AssociateArtifact(projectId, buildId, jobId, artifactName, ArtifactResourceTypes.Container, fileContainerFullPath, propertiesDictionary, token); + + context.Output($"Associated artifact {artifactName} ({artifact.Id}) with run #{buildId}"); + context.Debug($"Associated artifact using v1 endpoint"); + } } } } diff --git a/src/Sdk/PipelinesWebApi/Contracts/ActionsStorageArtifact.cs b/src/Sdk/PipelinesWebApi/Contracts/ActionsStorageArtifact.cs new file mode 100644 index 000000000..097f435ce --- /dev/null +++ b/src/Sdk/PipelinesWebApi/Contracts/ActionsStorageArtifact.cs @@ -0,0 +1,44 @@ +using System.Runtime.Serialization; +using GitHub.Services.WebApi; + +namespace GitHub.Actions.Pipelines.WebApi +{ + [DataContract] + public class ActionsStorageArtifact : Artifact + { + public ActionsStorageArtifact() + : base(ArtifactType.Actions_Storage) + { + } + + /// + /// File Container ID + /// + [DataMember] + public long ContainerId + { + get; + set; + } + + /// + /// Size of the file in bytes + /// + [DataMember] + public long Size + { + get; + set; + } + + /// + /// Signed content url for downloading the artifact + /// + [DataMember] + public SignedUrl SignedContent + { + get; + set; + } + } +} diff --git a/src/Sdk/PipelinesWebApi/Contracts/Artifact.cs b/src/Sdk/PipelinesWebApi/Contracts/Artifact.cs new file mode 100644 index 000000000..23eb33c11 --- /dev/null +++ b/src/Sdk/PipelinesWebApi/Contracts/Artifact.cs @@ -0,0 +1,46 @@ +using System.Runtime.Serialization; +using GitHub.Actions.Pipelines.WebApi.Contracts; +using Newtonsoft.Json; + +namespace GitHub.Actions.Pipelines.WebApi +{ + [DataContract] + [KnownType(typeof(ActionsStorageArtifact))] + [JsonConverter(typeof(ArtifactJsonConverter))] + public class Artifact + { + public Artifact(ArtifactType type) + { + Type = type; + } + + /// + /// The type of the artifact. + /// + [DataMember] + public ArtifactType Type + { + get; + } + + /// + /// The name of the artifact. + /// + [DataMember] + public string Name + { + get; + set; + } + + /// + /// Self-referential url + /// + [DataMember] + public string Url + { + get; + set; + } + } +} diff --git a/src/Sdk/PipelinesWebApi/Contracts/ArtifactBaseJsonConverter.cs b/src/Sdk/PipelinesWebApi/Contracts/ArtifactBaseJsonConverter.cs new file mode 100644 index 000000000..3a3ad979d --- /dev/null +++ b/src/Sdk/PipelinesWebApi/Contracts/ArtifactBaseJsonConverter.cs @@ -0,0 +1,85 @@ +using System; +using System.Reflection; +using GitHub.Services.WebApi; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Serialization; + +namespace GitHub.Actions.Pipelines.WebApi +{ + public abstract class ArtifactBaseJsonConverter : VssSecureJsonConverter where T : class + { + public override bool CanConvert(Type objectType) + { + return typeof(T).GetTypeInfo().IsAssignableFrom(objectType.GetTypeInfo()); + } + + // by returning false, the converter doesn't take part in writes + // which means we use the default serialization logic + public override bool CanWrite + { + get + { + return false; + } + } + + protected abstract T Create(Type objectType); + + protected abstract T Create(ArtifactType type); + + public override object ReadJson( + JsonReader reader, + Type objectType, + object existingValue, + JsonSerializer serializer) + { + if (reader.TokenType != JsonToken.StartObject) + { + return null; + } + + var contract = serializer.ContractResolver.ResolveContract(objectType) as JsonObjectContract; + if (contract == null) + { + return existingValue; + } + + // if objectType is one of our known types, we can ignore the type property + T targetObject = Create(objectType); + + // read the data into a JObject so we can look at it + var value = JObject.Load(reader); + + if (targetObject == null) + { + // use the Type property + var typeProperty = contract.Properties.GetClosestMatchProperty("Type"); + if (typeProperty == null) + { + // we don't know the type. just bail + return existingValue; + } + + if (!value.TryGetValue(typeProperty.PropertyName, StringComparison.OrdinalIgnoreCase, out var typeValue)) + { + // a type property exists on the contract, but the JObject has no value for it + return existingValue; + } + + var type = UnknownEnum.Parse(typeValue.ToString()); + targetObject = Create(type); + } + + if (targetObject != null) + { + using (var objectReader = value.CreateReader()) + { + serializer.Populate(objectReader, targetObject); + } + } + + return targetObject; + } + } +} diff --git a/src/Sdk/PipelinesWebApi/Contracts/ArtifactJsonConverter.cs b/src/Sdk/PipelinesWebApi/Contracts/ArtifactJsonConverter.cs new file mode 100644 index 000000000..923f1fc28 --- /dev/null +++ b/src/Sdk/PipelinesWebApi/Contracts/ArtifactJsonConverter.cs @@ -0,0 +1,29 @@ +using System; + +namespace GitHub.Actions.Pipelines.WebApi.Contracts +{ + public class ArtifactJsonConverter : ArtifactBaseJsonConverter + { + protected override Artifact Create(Type objectType) + { + if (objectType == typeof(ActionsStorageArtifact)) + { + return new ActionsStorageArtifact(); + } + else + { + return null; + } + } + + protected override Artifact Create(ArtifactType type) + { + if (type == ArtifactType.Actions_Storage) + { + return new ActionsStorageArtifact(); + } + + return null; + } + } +} diff --git a/src/Sdk/PipelinesWebApi/Contracts/ArtifactType.cs b/src/Sdk/PipelinesWebApi/Contracts/ArtifactType.cs new file mode 100644 index 000000000..58beb8b3a --- /dev/null +++ b/src/Sdk/PipelinesWebApi/Contracts/ArtifactType.cs @@ -0,0 +1,14 @@ +using System; +using System.Runtime.Serialization; +using Newtonsoft.Json; + +namespace GitHub.Actions.Pipelines.WebApi +{ + [DataContract] + [JsonConverter(typeof(ArtifactTypeEnumJsonConverter))] + public enum ArtifactType + { + Unknown = 0, + Actions_Storage = 1 + } +} diff --git a/src/Sdk/PipelinesWebApi/Contracts/ArtifactTypeEnumJsonConverter.cs b/src/Sdk/PipelinesWebApi/Contracts/ArtifactTypeEnumJsonConverter.cs new file mode 100644 index 000000000..2fac308e6 --- /dev/null +++ b/src/Sdk/PipelinesWebApi/Contracts/ArtifactTypeEnumJsonConverter.cs @@ -0,0 +1,25 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; +using System; +using System.Net.Http.Formatting; + +namespace GitHub.Actions.Pipelines.WebApi +{ + public class ArtifactTypeEnumJsonConverter : UnknownEnumJsonConverter + { + //json.net v12 exposes a "NamingStrategy" member that can do all this. We are at json.net v10 which only supports camel case. + //This is a poor man's way to fake it + public override void WriteJson(JsonWriter writer, object enumValue, JsonSerializer serializer) + { + var value = (ArtifactType)enumValue; + if (value == ArtifactType.Actions_Storage) + { + writer.WriteValue("actions_storage"); + } + else + { + base.WriteJson(writer, enumValue, serializer); + } + } + } +} diff --git a/src/Sdk/PipelinesWebApi/Contracts/CreateActionsStorageArtifactParameters.cs b/src/Sdk/PipelinesWebApi/Contracts/CreateActionsStorageArtifactParameters.cs new file mode 100644 index 000000000..4c17742f5 --- /dev/null +++ b/src/Sdk/PipelinesWebApi/Contracts/CreateActionsStorageArtifactParameters.cs @@ -0,0 +1,33 @@ +using System.Runtime.Serialization; + +namespace GitHub.Actions.Pipelines.WebApi +{ + [DataContract] + public class CreateActionsStorageArtifactParameters : CreateArtifactParameters + { + public CreateActionsStorageArtifactParameters() + : base(ArtifactType.Actions_Storage) + { + } + + /// + /// the id of the file container + /// + [DataMember] + public long ContainerId + { + get; + set; + } + + /// + /// Size of the file in bytes + /// + [DataMember] + public long Size + { + get; + set; + } + } +} diff --git a/src/Sdk/PipelinesWebApi/Contracts/CreateArtifactParameters.cs b/src/Sdk/PipelinesWebApi/Contracts/CreateArtifactParameters.cs new file mode 100644 index 000000000..3b8b5f6e8 --- /dev/null +++ b/src/Sdk/PipelinesWebApi/Contracts/CreateArtifactParameters.cs @@ -0,0 +1,35 @@ +using System.Runtime.Serialization; +using Newtonsoft.Json; + +namespace GitHub.Actions.Pipelines.WebApi +{ + [DataContract] + [KnownType(typeof(CreateActionsStorageArtifactParameters))] + [JsonConverter(typeof(CreateArtifactParametersJsonConverter))] + public class CreateArtifactParameters + { + protected CreateArtifactParameters(ArtifactType type) + { + Type = type; + } + + /// + /// The type of the artifact. + /// + [DataMember] + public ArtifactType Type + { + get; + } + + /// + /// The name of the artifact. + /// + [DataMember] + public string Name + { + get; + set; + } + } +} diff --git a/src/Sdk/PipelinesWebApi/Contracts/CreateArtifactParametersJsonConverter.cs b/src/Sdk/PipelinesWebApi/Contracts/CreateArtifactParametersJsonConverter.cs new file mode 100644 index 000000000..40949d1a7 --- /dev/null +++ b/src/Sdk/PipelinesWebApi/Contracts/CreateArtifactParametersJsonConverter.cs @@ -0,0 +1,29 @@ +using System; + +namespace GitHub.Actions.Pipelines.WebApi +{ + public class CreateArtifactParametersJsonConverter : ArtifactBaseJsonConverter + { + protected override CreateArtifactParameters Create(Type objectType) + { + if (objectType == typeof(CreateActionsStorageArtifactParameters)) + { + return new CreateActionsStorageArtifactParameters(); + } + else + { + return null; + } + } + + protected override CreateArtifactParameters Create(ArtifactType type) + { + if (type == ArtifactType.Actions_Storage) + { + return new CreateActionsStorageArtifactParameters(); + } + + return null; + } + } +} diff --git a/src/Sdk/PipelinesWebApi/Contracts/GetArtifactExpandOptions.cs b/src/Sdk/PipelinesWebApi/Contracts/GetArtifactExpandOptions.cs new file mode 100644 index 000000000..3494017a5 --- /dev/null +++ b/src/Sdk/PipelinesWebApi/Contracts/GetArtifactExpandOptions.cs @@ -0,0 +1,16 @@ +using System; +using System.ComponentModel; + +namespace GitHub.Actions.Pipelines.WebApi +{ + /// + /// $expand options for GetArtifact and ListArtifacts. + /// + [TypeConverter(typeof(KnownFlagsEnumTypeConverter))] + [Flags] + public enum GetArtifactExpandOptions + { + None = 0, + SignedContent = 1, + } +} diff --git a/src/Sdk/PipelinesWebApi/FlagsEnum.cs b/src/Sdk/PipelinesWebApi/FlagsEnum.cs new file mode 100644 index 000000000..abc1efa65 --- /dev/null +++ b/src/Sdk/PipelinesWebApi/FlagsEnum.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Runtime.Serialization; +using GitHub.Services.Common; + +namespace GitHub.Actions.Pipelines.WebApi +{ + public static class FlagsEnum + { + public static TEnum ParseKnownFlags(string stringValue) where TEnum : System.Enum + { + return (TEnum)ParseKnownFlags(typeof(TEnum), stringValue); + } + + /// + /// Parse known enum flags in a comma-separated string. Unknown flags are ignored. Allows for degraded compatibility without serializing enums to integers. + /// + /// + /// Case insensitive. Both standard and EnumMemberAttribute names are parsed. + /// + /// Thrown if stringValue is null. + /// Thrown if a flag name is empty. + public static object ParseKnownFlags(Type enumType, string stringValue) + { + ArgumentUtility.CheckForNull(enumType, nameof(enumType)); + if (!enumType.IsEnum) + { + throw new ArgumentException(PipelinesWebApiResources.FlagEnumTypeRequired()); + } + + // Check for the flags attribute in debug. Skip this reflection in release. + Debug.Assert(enumType.GetCustomAttributes(typeof(FlagsAttribute), inherit: false).Any(), "FlagsEnum only intended for enums with the Flags attribute."); + + // The exception types below are based on Enum.TryParseEnum (http://index/?query=TryParseEnum&rightProject=mscorlib&file=system%5Cenum.cs&rightSymbol=bhaeh2vnegwo) + if (stringValue == null) + { + throw new ArgumentNullException(stringValue); + } + + if (String.IsNullOrWhiteSpace(stringValue)) + { + throw new ArgumentException(PipelinesWebApiResources.NonEmptyEnumElementsRequired(stringValue)); + } + + if (UInt64.TryParse(stringValue, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out ulong ulongValue)) + { + return Enum.Parse(enumType, stringValue); + } + + var enumNames = Enum.GetNames(enumType).ToHashSet(name => name, StringComparer.OrdinalIgnoreCase); + var enumMemberMappings = new Lazy>(() => + { + IDictionary mappings = null; + foreach (var field in enumType.GetFields()) + { + if (field.GetCustomAttributes(typeof(EnumMemberAttribute), false).FirstOrDefault() is EnumMemberAttribute enumMemberAttribute) + { + if (mappings == null) + { + mappings = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + mappings.Add(enumMemberAttribute.Value, field.GetValue(null).ToString()); + } + } + + return mappings; + }); + + var values = stringValue.Split(s_enumSeparatorCharArray); + + var matches = new List(); + for (int i = 0; i < values.Length; i++) + { + string value = values[i].Trim(); + + if (String.IsNullOrEmpty(value)) + { + throw new ArgumentException(PipelinesWebApiResources.NonEmptyEnumElementsRequired(stringValue)); + } + + if (enumNames.Contains(value)) + { + matches.Add(value); + } + else if (enumMemberMappings.Value != null && enumMemberMappings.Value.TryGetValue(value, out string matchingValue)) + { + matches.Add(matchingValue); + } + } + + if (!matches.Any()) + { + return Enum.Parse(enumType, "0"); + } + + string matchesString = String.Join(", ", matches); + return Enum.Parse(enumType, matchesString, ignoreCase: true); + } + + private static readonly char[] s_enumSeparatorCharArray = new char[] { ',' }; + } +} diff --git a/src/Sdk/PipelinesWebApi/Generated/PipelinesHttpClientBase.cs b/src/Sdk/PipelinesWebApi/Generated/PipelinesHttpClientBase.cs new file mode 100644 index 000000000..44d6fd258 --- /dev/null +++ b/src/Sdk/PipelinesWebApi/Generated/PipelinesHttpClientBase.cs @@ -0,0 +1,273 @@ +/* + * --------------------------------------------------------- + * Copyright(C) Microsoft Corporation. All rights reserved. + * --------------------------------------------------------- + * + * --------------------------------------------------------- + * Generated file, DO NOT EDIT + * --------------------------------------------------------- + * + * See following wiki page for instructions on how to regenerate: + * https://aka.ms/azure-devops-client-generation + * + * Configuration file: + * actions\client\webapi\clientgeneratorconfigs\pipelines.genclient.json + */ + +using System; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Formatting; +using System.Threading; +using System.Threading.Tasks; +using GitHub.Services.Common; +using GitHub.Services.WebApi; + +namespace GitHub.Actions.Pipelines.WebApi +{ + [ResourceArea(PipelinesArea.IdString)] + public abstract class PipelinesHttpClientBase : VssHttpClientBase + { + public PipelinesHttpClientBase(Uri baseUrl, VssCredentials credentials) + : base(baseUrl, credentials) + { + } + + public PipelinesHttpClientBase(Uri baseUrl, VssCredentials credentials, VssHttpRequestSettings settings) + : base(baseUrl, credentials, settings) + { + } + + public PipelinesHttpClientBase(Uri baseUrl, VssCredentials credentials, params DelegatingHandler[] handlers) + : base(baseUrl, credentials, handlers) + { + } + + public PipelinesHttpClientBase(Uri baseUrl, VssCredentials credentials, VssHttpRequestSettings settings, params DelegatingHandler[] handlers) + : base(baseUrl, credentials, settings, handlers) + { + } + + public PipelinesHttpClientBase(Uri baseUrl, HttpMessageHandler pipeline, bool disposeHandler) + : base(baseUrl, pipeline, disposeHandler) + { + } + + /// + /// [Preview API] Associates an artifact with a run. + /// + /// + /// The ID of the pipeline. + /// The ID of the run. + /// + /// The cancellation token to cancel operation. + public virtual Task CreateArtifactAsync( + CreateArtifactParameters createArtifactParameters, + int pipelineId, + int runId, + object userState = null, + CancellationToken cancellationToken = default) + { + HttpMethod httpMethod = new HttpMethod("POST"); + Guid locationId = new Guid("85023071-bd5e-4438-89b0-2a5bf362a19d"); + object routeValues = new { pipelineId = pipelineId, runId = runId }; + HttpContent content = new ObjectContent(createArtifactParameters, new VssJsonMediaTypeFormatter(true)); + + return SendAsync( + httpMethod, + locationId, + routeValues: routeValues, + version: new ApiResourceVersion(6.0, 1), + userState: userState, + cancellationToken: cancellationToken, + content: content); + } + + /// + /// [Preview API] Associates an artifact with a run. + /// + /// + /// Project ID or project name + /// The ID of the pipeline. + /// The ID of the run. + /// + /// The cancellation token to cancel operation. + public virtual Task CreateArtifactAsync( + CreateArtifactParameters createArtifactParameters, + string project, + int pipelineId, + int runId, + object userState = null, + CancellationToken cancellationToken = default) + { + HttpMethod httpMethod = new HttpMethod("POST"); + Guid locationId = new Guid("85023071-bd5e-4438-89b0-2a5bf362a19d"); + object routeValues = new { project = project, pipelineId = pipelineId, runId = runId }; + HttpContent content = new ObjectContent(createArtifactParameters, new VssJsonMediaTypeFormatter(true)); + + return SendAsync( + httpMethod, + locationId, + routeValues: routeValues, + version: new ApiResourceVersion(6.0, 1), + userState: userState, + cancellationToken: cancellationToken, + content: content); + } + + /// + /// [Preview API] Associates an artifact with a run. + /// + /// + /// Project ID + /// The ID of the pipeline. + /// The ID of the run. + /// + /// The cancellation token to cancel operation. + public virtual Task CreateArtifactAsync( + CreateArtifactParameters createArtifactParameters, + Guid project, + int pipelineId, + int runId, + object userState = null, + CancellationToken cancellationToken = default) + { + HttpMethod httpMethod = new HttpMethod("POST"); + Guid locationId = new Guid("85023071-bd5e-4438-89b0-2a5bf362a19d"); + object routeValues = new { project = project, pipelineId = pipelineId, runId = runId }; + HttpContent content = new ObjectContent(createArtifactParameters, new VssJsonMediaTypeFormatter(true)); + + return SendAsync( + httpMethod, + locationId, + routeValues: routeValues, + version: new ApiResourceVersion(6.0, 1), + userState: userState, + cancellationToken: cancellationToken, + content: content); + } + + /// + /// [Preview API] Get a specific artifact + /// + /// Project ID or project name + /// + /// + /// + /// + /// + /// The cancellation token to cancel operation. + public virtual Task GetArtifactAsync( + string project, + int pipelineId, + int runId, + string artifactName, + GetArtifactExpandOptions? expand = null, + object userState = null, + CancellationToken cancellationToken = default) + { + HttpMethod httpMethod = new HttpMethod("GET"); + Guid locationId = new Guid("85023071-bd5e-4438-89b0-2a5bf362a19d"); + object routeValues = new { project = project, pipelineId = pipelineId, runId = runId }; + + List> queryParams = new List>(); + queryParams.Add("artifactName", artifactName); + if (expand != null) + { + queryParams.Add("$expand", expand.Value.ToString()); + } + + return SendAsync( + httpMethod, + locationId, + routeValues: routeValues, + version: new ApiResourceVersion(6.0, 1), + queryParameters: queryParams, + userState: userState, + cancellationToken: cancellationToken); + } + + /// + /// [Preview API] Get a specific artifact + /// + /// Project ID + /// + /// + /// + /// + /// + /// The cancellation token to cancel operation. + public virtual Task GetArtifactAsync( + Guid project, + int pipelineId, + int runId, + string artifactName, + GetArtifactExpandOptions? expand = null, + object userState = null, + CancellationToken cancellationToken = default) + { + HttpMethod httpMethod = new HttpMethod("GET"); + Guid locationId = new Guid("85023071-bd5e-4438-89b0-2a5bf362a19d"); + object routeValues = new { project = project, pipelineId = pipelineId, runId = runId }; + + List> queryParams = new List>(); + queryParams.Add("artifactName", artifactName); + if (expand != null) + { + queryParams.Add("$expand", expand.Value.ToString()); + } + + return SendAsync( + httpMethod, + locationId, + routeValues: routeValues, + version: new ApiResourceVersion(6.0, 1), + queryParameters: queryParams, + userState: userState, + cancellationToken: cancellationToken); + } + + /// + /// [Preview API] Get a specific artifact + /// + /// + /// + /// + /// + /// + /// The cancellation token to cancel operation. + public virtual Task GetArtifactAsync( + int pipelineId, + int runId, + string artifactName, + GetArtifactExpandOptions? expand = null, + object userState = null, + CancellationToken cancellationToken = default) + { + HttpMethod httpMethod = new HttpMethod("GET"); + Guid locationId = new Guid("85023071-bd5e-4438-89b0-2a5bf362a19d"); + object routeValues = new { pipelineId = pipelineId, runId = runId }; + + List> queryParams = new List>(); + queryParams.Add("artifactName", artifactName); + if (expand != null) + { + queryParams.Add("$expand", expand.Value.ToString()); + } + + return SendAsync( + httpMethod, + locationId, + routeValues: routeValues, + version: new ApiResourceVersion(6.0, 1), + queryParameters: queryParams, + userState: userState, + cancellationToken: cancellationToken); + } + } +} diff --git a/src/Sdk/PipelinesWebApi/KnownFlagsEnumTypeConverter.cs b/src/Sdk/PipelinesWebApi/KnownFlagsEnumTypeConverter.cs new file mode 100644 index 000000000..786178565 --- /dev/null +++ b/src/Sdk/PipelinesWebApi/KnownFlagsEnumTypeConverter.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Globalization; + +namespace GitHub.Actions.Pipelines.WebApi +{ + /// + /// Parses known enum flags in a comma-separated string. Unknown flags are ignored. Allows for degraded compatibility without serializing enums to integer values. + /// + /// + /// Case insensitive. Both standard and EnumMemberAttribute names are parsed. + /// json deserialization doesn't happen for query parameters :) + /// + public class KnownFlagsEnumTypeConverter : EnumConverter + { + public KnownFlagsEnumTypeConverter(Type type) + : base(type) + { + } + + public override bool CanConvertFrom(ITypeDescriptorContext context, Type sourceType) + { + return sourceType == typeof(string) || base.CanConvertFrom(context, sourceType); + } + + /// Thrown if a flag name is empty. + public override object ConvertFrom(ITypeDescriptorContext context, CultureInfo culture, object value) + { + if (value is string stringValue) + { + try + { + return FlagsEnum.ParseKnownFlags(EnumType, stringValue); + } + catch (Exception ex) + { + // Matches the exception type thrown by EnumTypeConverter. + throw new FormatException(PipelinesWebApiResources.InvalidFlagsEnumValue(stringValue, EnumType), ex); + } + } + return base.ConvertFrom(context, culture, value); + } + } +} diff --git a/src/Sdk/PipelinesWebApi/PipelinesHttpClient.cs b/src/Sdk/PipelinesWebApi/PipelinesHttpClient.cs new file mode 100644 index 000000000..5b854ed53 --- /dev/null +++ b/src/Sdk/PipelinesWebApi/PipelinesHttpClient.cs @@ -0,0 +1,36 @@ +using System; +using System.Net.Http; +using GitHub.Services.Common; +using GitHub.Services.WebApi; + +namespace GitHub.Actions.Pipelines.WebApi +{ + [ResourceArea(PipelinesArea.IdString)] + public class PipelinesHttpClient : PipelinesHttpClientBase + { + public PipelinesHttpClient(Uri baseUrl, VssCredentials credentials) + : base(baseUrl, credentials) + { + } + + public PipelinesHttpClient(Uri baseUrl, VssCredentials credentials, VssHttpRequestSettings settings) + : base(baseUrl, credentials, settings) + { + } + + public PipelinesHttpClient(Uri baseUrl, VssCredentials credentials, params DelegatingHandler[] handlers) + : base(baseUrl, credentials, handlers) + { + } + + public PipelinesHttpClient(Uri baseUrl, VssCredentials credentials, VssHttpRequestSettings settings, params DelegatingHandler[] handlers) + : base(baseUrl, credentials, settings, handlers) + { + } + + public PipelinesHttpClient(Uri baseUrl, HttpMessageHandler pipeline, bool disposeHandler) + : base(baseUrl, pipeline, disposeHandler) + { + } + } +} diff --git a/src/Sdk/PipelinesWebApi/PipelinesResourceIds.cs b/src/Sdk/PipelinesWebApi/PipelinesResourceIds.cs new file mode 100644 index 000000000..0cc620b07 --- /dev/null +++ b/src/Sdk/PipelinesWebApi/PipelinesResourceIds.cs @@ -0,0 +1,74 @@ +using System; + +namespace GitHub.Actions.Pipelines.WebApi +{ + public static class PipelinesArea + { + public const string Name = "pipelines"; + public const string IdString = "2e0bf237-8973-4ec9-a581-9c3d679d1776"; + public static readonly Guid Id = new Guid(PipelinesArea.IdString); + } + + public static class PipelinesResources + { + public static class Artifacts + { + public const string Name = "artifacts"; + public static readonly Guid Id = new Guid("85023071-BD5E-4438-89B0-2A5BF362A19D"); + } + + public static class PipelineOrgs + { + public const string Name = "orgs"; + public static readonly Guid Id = new Guid("CD70BA1A-D59A-4E0B-9934-97998159CCC8"); + } + + public static class Logs + { + public const string Name = "logs"; + public static readonly Guid Id = new Guid("fb1b6d27-3957-43d5-a14b-a2d70403e545"); + } + + public static class Pipelines + { + public const string Name = "pipelines"; + public static readonly Guid Id = new Guid("28e1305e-2afe-47bf-abaf-cbb0e6a91988"); + } + + public static class Reputations + { + public const string Name = "reputations"; + public static readonly Guid Id = new Guid("ABA353B0-46FB-4885-88C5-391C6B6382B3"); + } + + public static class Runs + { + public const string Name = "runs"; + public static readonly Guid Id = new Guid("7859261e-d2e9-4a68-b820-a5d84cc5bb3d"); + } + + public static class SignalR + { + public const string Name = "signalr"; + public static readonly Guid Id = new Guid("1FFE4916-AC72-4566-ADD0-9BAB31E44FCF"); + } + + public static class SignedArtifactsContent + { + public const string Name = "signedartifactscontent"; + public static readonly Guid Id = new Guid("6B2AC16F-CD00-4DF9-A13B-3A1CC8AFB188"); + } + + public static class SignedLogContent + { + public const string Name = "signedlogcontent"; + public static readonly Guid Id = new Guid("74f99e32-e2c4-44f4-93dc-dec0bca530a5"); + } + + public static class SignalRLive + { + public const string Name = "live"; + public static readonly Guid Id = new Guid("C41B3775-6D50-48BD-B261-42DA7F0F1BA0"); + } + } +} diff --git a/src/Sdk/PipelinesWebApi/UnknownEnum.cs b/src/Sdk/PipelinesWebApi/UnknownEnum.cs new file mode 100644 index 000000000..48ab699eb --- /dev/null +++ b/src/Sdk/PipelinesWebApi/UnknownEnum.cs @@ -0,0 +1,45 @@ +using System; +using System.Linq; +using System.Runtime.Serialization; + +namespace GitHub.Actions.Pipelines.WebApi +{ + public static class UnknownEnum + { + public static T Parse(string stringValue) + { + return (T)Parse(typeof(T), stringValue); + } + + public static object Parse(Type enumType, string stringValue) + { + var underlyingType = Nullable.GetUnderlyingType(enumType); + enumType = underlyingType != null ? underlyingType : enumType; + + var names = Enum.GetNames(enumType); + if (!string.IsNullOrEmpty(stringValue)) + { + var match = names.FirstOrDefault(name => string.Equals(name, stringValue, StringComparison.OrdinalIgnoreCase)); + if (match != null) + { + return Enum.Parse(enumType, match); + } + + // maybe we have an enum member with an EnumMember attribute specifying a custom name + foreach (var field in enumType.GetFields()) + { + var enumMemberAttribute = field.GetCustomAttributes(typeof(EnumMemberAttribute), false).FirstOrDefault() as EnumMemberAttribute; + if (enumMemberAttribute != null && string.Equals(enumMemberAttribute.Value, stringValue, StringComparison.OrdinalIgnoreCase)) + { + // we already have the field, no need to do enum.parse on it + return field.GetValue(null); + } + } + } + + return Enum.Parse(enumType, UnknownName); + } + + private const string UnknownName = "Unknown"; + } +} diff --git a/src/Sdk/PipelinesWebApi/UnknownEnumJsonConverter.cs b/src/Sdk/PipelinesWebApi/UnknownEnumJsonConverter.cs new file mode 100644 index 000000000..f27ae0ab7 --- /dev/null +++ b/src/Sdk/PipelinesWebApi/UnknownEnumJsonConverter.cs @@ -0,0 +1,53 @@ +using System; +using System.Linq; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace GitHub.Actions.Pipelines.WebApi +{ + public class UnknownEnumJsonConverter : StringEnumConverter + { + public UnknownEnumJsonConverter() + { + this.CamelCaseText = true; + } + + public override bool CanConvert(Type objectType) + { + // we require one member to be named "Unknown" + return objectType.IsEnum && Enum.GetNames(objectType).Any(name => string.Equals(name, UnknownName, StringComparison.OrdinalIgnoreCase)); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + // Newtonsoft doesn't call CanConvert if you specify the converter using a JsonConverter attribute + // they just assume you know what you're doing :) + if (!CanConvert(objectType)) + { + // if there's no Unknown value, fall back to the StringEnumConverter behavior + return base.ReadJson(reader, objectType, existingValue, serializer); + } + + if (reader.TokenType == JsonToken.Integer) + { + var intValue = Convert.ToInt32(reader.Value); + var values = (int[])Enum.GetValues(objectType); + if (values.Contains(intValue)) + { + return Enum.Parse(objectType, intValue.ToString()); + } + } + + if (reader.TokenType == JsonToken.String) + { + var stringValue = reader.Value.ToString(); + return UnknownEnum.Parse(objectType, stringValue); + } + + // we know there's an Unknown value because CanConvert returned true + return Enum.Parse(objectType, UnknownName); + } + + private const string UnknownName = "Unknown"; + } +} diff --git a/src/Sdk/Resources/PipelinesWebApiResources.g.cs b/src/Sdk/Resources/PipelinesWebApiResources.g.cs new file mode 100644 index 000000000..90bbddf30 --- /dev/null +++ b/src/Sdk/Resources/PipelinesWebApiResources.g.cs @@ -0,0 +1,26 @@ +using System.Globalization; + +namespace GitHub.Actions.Pipelines.WebApi +{ + public static class PipelinesWebApiResources + { + + public static string FlagEnumTypeRequired() + { + const string Format = @"Invalid type. An enum type with the Flags attribute must be supplied."; + return Format; + } + + public static string InvalidFlagsEnumValue(object arg0, object arg1) + { + const string Format = @"'{0}' is not a valid value for {1}"; + return string.Format(CultureInfo.CurrentCulture, Format, arg0, arg1); + } + + public static string NonEmptyEnumElementsRequired(object arg0) + { + const string Format = @"Each comma separated enum value must be non-empty: {0}"; + return string.Format(CultureInfo.CurrentCulture, Format, arg0); + } + } +} diff --git a/src/Sdk/WebApi/WebApi/Contracts/SignedUrl/SignedUrl.cs b/src/Sdk/WebApi/WebApi/Contracts/SignedUrl/SignedUrl.cs new file mode 100644 index 000000000..1802e7fe2 --- /dev/null +++ b/src/Sdk/WebApi/WebApi/Contracts/SignedUrl/SignedUrl.cs @@ -0,0 +1,18 @@ +using System; +using System.Runtime.Serialization; + +namespace GitHub.Services.WebApi +{ + /// + /// A signed url allowing limited-time anonymous access to private resources. + /// + [DataContract] + public class SignedUrl + { + [DataMember] + public string Url { get; set; } + + [DataMember] + public DateTime SignatureExpires { get; set; } + } +}