Report job has infra failure to run-service (#4073)

This commit is contained in:
Tingluo Huang
2025-10-13 16:21:32 -04:00
committed by GitHub
parent afe4fc8446
commit 1eb15f28a7
9 changed files with 23 additions and 10 deletions

View File

@@ -30,6 +30,7 @@ namespace GitHub.Runner.Common
string environmentUrl, string environmentUrl,
IList<Telemetry> telemetry, IList<Telemetry> telemetry,
string billingOwnerId, string billingOwnerId,
string infrastructureFailureCategory,
CancellationToken token); CancellationToken token);
Task<RenewJobResponse> RenewJobAsync(Guid planId, Guid jobId, CancellationToken token); Task<RenewJobResponse> RenewJobAsync(Guid planId, Guid jobId, CancellationToken token);
@@ -80,11 +81,12 @@ namespace GitHub.Runner.Common
string environmentUrl, string environmentUrl,
IList<Telemetry> telemetry, IList<Telemetry> telemetry,
string billingOwnerId, string billingOwnerId,
string infrastructureFailureCategory,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
CheckConnection(); CheckConnection();
return RetryRequest( return RetryRequest(
async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, telemetry, billingOwnerId, cancellationToken), cancellationToken, async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, telemetry, billingOwnerId, infrastructureFailureCategory, cancellationToken), cancellationToken,
shouldRetry: ex => shouldRetry: ex =>
ex is not VssUnauthorizedException && // HTTP status 401 ex is not VssUnauthorizedException && // HTTP status 401
ex is not TaskOrchestrationJobNotFoundException); // HTTP status 404 ex is not TaskOrchestrationJobNotFoundException); // HTTP status 404

View File

@@ -1211,7 +1211,7 @@ namespace GitHub.Runner.Listener
jobAnnotations.Add(annotation.Value); jobAnnotations.Add(annotation.Value);
} }
await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, telemetry: null, billingOwnerId: message.BillingOwnerId, CancellationToken.None); await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, telemetry: null, billingOwnerId: message.BillingOwnerId, infrastructureFailureCategory: null, CancellationToken.None);
} }
catch (Exception ex) catch (Exception ex)
{ {

View File

@@ -111,7 +111,7 @@ namespace GitHub.Runner.Worker
{ {
// Log the error and fail the PrepareActionsAsync Initialization. // Log the error and fail the PrepareActionsAsync Initialization.
Trace.Error($"Caught exception from PrepareActionsAsync Initialization: {ex}"); Trace.Error($"Caught exception from PrepareActionsAsync Initialization: {ex}");
executionContext.InfrastructureError(ex.Message); executionContext.InfrastructureError(ex.Message, category: "resolve_action");
executionContext.Result = TaskResult.Failed; executionContext.Result = TaskResult.Failed;
throw; throw;
} }
@@ -119,7 +119,7 @@ namespace GitHub.Runner.Worker
{ {
// Log the error and fail the PrepareActionsAsync Initialization. // Log the error and fail the PrepareActionsAsync Initialization.
Trace.Error($"Caught exception from PrepareActionsAsync Initialization: {ex}"); Trace.Error($"Caught exception from PrepareActionsAsync Initialization: {ex}");
executionContext.InfrastructureError(ex.Message); executionContext.InfrastructureError(ex.Message, category: "invalid_action_download");
executionContext.Result = TaskResult.Failed; executionContext.Result = TaskResult.Failed;
throw; throw;
} }
@@ -777,15 +777,15 @@ namespace GitHub.Runner.Worker
IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken); IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
Directory.CreateDirectory(destDirectory); Directory.CreateDirectory(destDirectory);
if (downloadInfo.PackageDetails != null) if (downloadInfo.PackageDetails != null)
{ {
executionContext.Output($"##[group]Download immutable action package '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}'"); executionContext.Output($"##[group]Download immutable action package '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}'");
executionContext.Output($"Version: {downloadInfo.PackageDetails.Version}"); executionContext.Output($"Version: {downloadInfo.PackageDetails.Version}");
executionContext.Output($"Digest: {downloadInfo.PackageDetails.ManifestDigest}"); executionContext.Output($"Digest: {downloadInfo.PackageDetails.ManifestDigest}");
executionContext.Output($"Source commit SHA: {downloadInfo.ResolvedSha}"); executionContext.Output($"Source commit SHA: {downloadInfo.ResolvedSha}");
executionContext.Output("##[endgroup]"); executionContext.Output("##[endgroup]");
} }
else else
{ {
executionContext.Output($"Download action repository '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}' (SHA:{downloadInfo.ResolvedSha})"); executionContext.Output($"Download action repository '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}' (SHA:{downloadInfo.ResolvedSha})");
} }

View File

@@ -522,6 +522,10 @@ namespace GitHub.Runner.Worker
if (annotation != null) if (annotation != null)
{ {
stepResult.Annotations.Add(annotation.Value); stepResult.Annotations.Add(annotation.Value);
if (annotation.Value.IsInfrastructureIssue && string.IsNullOrEmpty(Global.InfrastructureFailureCategory))
{
Global.InfrastructureFailureCategory = issue.Category;
}
} }
}); });
@@ -1335,9 +1339,9 @@ namespace GitHub.Runner.Worker
} }
// Do not add a format string overload. See comment on ExecutionContext.Write(). // Do not add a format string overload. See comment on ExecutionContext.Write().
public static void InfrastructureError(this IExecutionContext context, string message) public static void InfrastructureError(this IExecutionContext context, string message, string category = null)
{ {
var issue = new Issue() { Type = IssueType.Error, Message = message, IsInfrastructureIssue = true }; var issue = new Issue() { Type = IssueType.Error, Message = message, IsInfrastructureIssue = true, Category = category };
context.AddIssue(issue, ExecutionContextLogOptions.Default); context.AddIssue(issue, ExecutionContextLogOptions.Default);
} }

View File

@@ -27,6 +27,7 @@ namespace GitHub.Runner.Worker
public StepsContext StepsContext { get; set; } public StepsContext StepsContext { get; set; }
public Variables Variables { get; set; } public Variables Variables { get; set; }
public bool WriteDebug { get; set; } public bool WriteDebug { get; set; }
public string InfrastructureFailureCategory { get; set; }
public JObject ContainerHookState { get; set; } public JObject ContainerHookState { get; set; }
} }
} }

View File

@@ -321,7 +321,7 @@ namespace GitHub.Runner.Worker
{ {
try try
{ {
await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, telemetry, billingOwnerId: message.BillingOwnerId, default); await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, telemetry, billingOwnerId: message.BillingOwnerId, infrastructureFailureCategory: jobContext.Global.InfrastructureFailureCategory, default);
return result; return result;
} }
catch (VssUnauthorizedException ex) catch (VssUnauthorizedException ex)

View File

@@ -35,5 +35,8 @@ namespace GitHub.Actions.RunService.WebApi
[DataMember(Name = "billingOwnerId", EmitDefaultValue = false)] [DataMember(Name = "billingOwnerId", EmitDefaultValue = false)]
public string BillingOwnerId { get; set; } public string BillingOwnerId { get; set; }
[DataMember(Name = "infrastructureFailureCategory", EmitDefaultValue = false)]
public string InfrastructureFailureCategory { get; set; }
} }
} }

View File

@@ -42,6 +42,7 @@ namespace Sdk.RSWebApi.Contracts
StartColumn = columnNumber, StartColumn = columnNumber,
EndColumn = endColumnNumber, EndColumn = endColumnNumber,
StepNumber = stepNumber, StepNumber = stepNumber,
IsInfrastructureIssue = issue.IsInfrastructureIssue ?? false
}; };
} }

View File

@@ -131,6 +131,7 @@ namespace GitHub.Actions.RunService.WebApi
string environmentUrl, string environmentUrl,
IList<Telemetry> telemetry, IList<Telemetry> telemetry,
string billingOwnerId, string billingOwnerId,
string infrastructureFailureCategory,
CancellationToken cancellationToken = default) CancellationToken cancellationToken = default)
{ {
HttpMethod httpMethod = new HttpMethod("POST"); HttpMethod httpMethod = new HttpMethod("POST");
@@ -145,6 +146,7 @@ namespace GitHub.Actions.RunService.WebApi
EnvironmentUrl = environmentUrl, EnvironmentUrl = environmentUrl,
Telemetry = telemetry, Telemetry = telemetry,
BillingOwnerId = billingOwnerId, BillingOwnerId = billingOwnerId,
InfrastructureFailureCategory = infrastructureFailureCategory
}; };
requestUri = new Uri(requestUri, "completejob"); requestUri = new Uri(requestUri, "completejob");