Wait for job record updated before running steps. (#1320)

* Wait for job record updated before running steps.

* only oidc
This commit is contained in:
Tingluo Huang
2021-09-09 21:55:15 -04:00
committed by GitHub
parent 4359dd605b
commit 72fa2a8a0d
2 changed files with 24 additions and 2 deletions

View File

@@ -15,6 +15,7 @@ namespace GitHub.Runner.Common
[ServiceLocator(Default = typeof(JobServerQueue))] [ServiceLocator(Default = typeof(JobServerQueue))]
public interface IJobServerQueue : IRunnerService, IThrottlingReporter public interface IJobServerQueue : IRunnerService, IThrottlingReporter
{ {
TaskCompletionSource<int> JobRecordUpdated { get; }
event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling; event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling;
Task ShutdownAsync(); Task ShutdownAsync();
void Start(Pipelines.AgentJobRequestMessage jobRequest); void Start(Pipelines.AgentJobRequestMessage jobRequest);
@@ -62,8 +63,11 @@ namespace GitHub.Runner.Common
private IJobServer _jobServer; private IJobServer _jobServer;
private Task[] _allDequeueTasks; private Task[] _allDequeueTasks;
private readonly TaskCompletionSource<int> _jobCompletionSource = new TaskCompletionSource<int>(); private readonly TaskCompletionSource<int> _jobCompletionSource = new TaskCompletionSource<int>();
private readonly TaskCompletionSource<int> _jobRecordUpdated = new TaskCompletionSource<int>();
private bool _queueInProcess = false; private bool _queueInProcess = false;
public TaskCompletionSource<int> JobRecordUpdated => _jobRecordUpdated;
public event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling; public event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling;
// Web console dequeue will start with process queue every 250ms for the first 60*4 times (~60 seconds). // Web console dequeue will start with process queue every 250ms for the first 60*4 times (~60 seconds).
@@ -455,6 +459,14 @@ namespace GitHub.Runner.Common
{ {
Trace.Verbose("Cleanup buffered timeline record for timeline: {0}.", update.TimelineId); Trace.Verbose("Cleanup buffered timeline record for timeline: {0}.", update.TimelineId);
} }
if (!_jobRecordUpdated.Task.IsCompleted &&
update.PendingRecords.Any(x => x.Id == _jobTimelineRecordId && x.State != null))
{
// We have changed the state of the job
Trace.Info("Job timeline record has been updated for the first time.");
_jobRecordUpdated.TrySetResult(0);
}
} }
catch (Exception ex) catch (Exception ex)
{ {

View File

@@ -145,6 +145,16 @@ namespace GitHub.Runner.Worker
Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'"); Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}"); HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}");
if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) &&
!string.IsNullOrEmpty(generateIdTokenUrl))
{
// Server won't issue ID_TOKEN for non-inprogress job.
// If the job is trying to use OIDC feature, we want the job to be marked as in-progress before running any customer's steps as much as we can.
// Timeline record update background process runs every 500ms, so delay 1000ms is enough for most of the cases
Trace.Info($"Waiting for job to be marked as started.");
await Task.WhenAny(_jobServerQueue.JobRecordUpdated.Task, Task.Delay(1000));
}
// Run all job steps // Run all job steps
Trace.Info("Run all job steps."); Trace.Info("Run all job steps.");
var stepsRunner = HostContext.GetService<IStepsRunner>(); var stepsRunner = HostContext.GetService<IStepsRunner>();