actions/actions-runner-controller
1
0
Fork 0
mirror of https://github.com/actions/actions-runner-controller.git synced 2025-12-12 20:46:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix scale-from-zero to retain the reg-only runner until other pods come up (#523)

Browse Source 
Fixes #516
This commit is contained in:
Yusuke Kuoka
2021-05-05 12:13:51 +09:00
committed by GitHub
parent cb54864387
commit e7020c7c0f
3 changed files with 85 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
controllers/runnerreplicaset_controller.go
View File

@@ -68,65 +68,6 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
return ctrl.Result{}, nil
}
registrationOnlyRunnerNeeded := rs.Spec.Replicas != nil && *rs.Spec.Replicas == 0
registrationOnlyRunner := v1alpha1.Runner{}
registrationOnlyRunnerNsName := req.NamespacedName
registrationOnlyRunnerNsName.Name = registrationOnlyRunnerNameFor(rs.Name)
registrationOnlyRunnerExists := false
if err := r.Get(
ctx,
registrationOnlyRunnerNsName,
&registrationOnlyRunner,
); err != nil {
if !kerrors.IsNotFound(err) {
return ctrl.Result{}, err
}
} else {
registrationOnlyRunnerExists = true
}
if registrationOnlyRunnerNeeded {
if registrationOnlyRunnerExists {
if registrationOnlyRunner.Status.Phase == "" {
log.Info("Still waiting for the registration-only runner to be registered")
return ctrl.Result{}, nil
}
} else {
// A registration-only runner does not exist and is needed, hence create it.
runnerForScaleFromToZero, err := r.newRunner(rs)
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to create runner for scale from/to zero: %v", err)
}
runnerForScaleFromToZero.ObjectMeta.Name = registrationOnlyRunnerNsName.Name
runnerForScaleFromToZero.ObjectMeta.GenerateName = ""
runnerForScaleFromToZero.ObjectMeta.Labels = nil
metav1.SetMetaDataAnnotation(&runnerForScaleFromToZero.ObjectMeta, annotationKeyRegistrationOnly, "true")
if err := r.Client.Create(ctx, &runnerForScaleFromToZero); err != nil {
log.Error(err, "Failed to create runner for scale from/to zero")
return ctrl.Result{}, err
}
// We can continue to deleting runner pods only after the
// registration-only runner gets registered.
return ctrl.Result{}, nil
}
} else {
// A registration-only runner exists and is not needed, hence delete it.
if registrationOnlyRunnerExists {
if err := r.Client.Delete(ctx, &registrationOnlyRunner); err != nil {
log.Error(err, "Retrying soon because we failed to delete registration-only runner")
return ctrl.Result{Requeue: true}, nil
}
}
}
selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
if err != nil {
return ctrl.Result{}, err
@@ -173,6 +114,71 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
desired = 1
}
registrationOnlyRunnerNsName := req.NamespacedName
registrationOnlyRunnerNsName.Name = registrationOnlyRunnerNameFor(rs.Name)
registrationOnlyRunner := v1alpha1.Runner{}
registrationOnlyRunnerExists := false
if err := r.Get(
ctx,
registrationOnlyRunnerNsName,
&registrationOnlyRunner,
); err != nil {
if !kerrors.IsNotFound(err) {
return ctrl.Result{}, err
}
} else {
registrationOnlyRunnerExists = true
}
// On scale to zero, we must have fully registered registration-only runner before we start deleting other runners, hence `desired == 0`
// On scale from zero, we must retain the registratoin-only runner until one or more other runners get registered, hence `registrationOnlyRunnerExists && available == 0`.
// On RunnerReplicaSet creation, it have always 0 replics and no registration-only runner.
// In this case We don't need to bother creating a registration-only runner which gets deleted soon after we have 1 or more available repolicas,
// hence it's not `available == 0`, but `registrationOnlyRunnerExists && available == 0`.
// See https://github.com/actions-runner-controller/actions-runner-controller/issues/516
registrationOnlyRunnerNeeded := desired == 0 || (registrationOnlyRunnerExists && available == 0)
if registrationOnlyRunnerNeeded {
if registrationOnlyRunnerExists {
if registrationOnlyRunner.Status.Phase == "" {
log.Info("Still waiting for the registration-only runner to be registered")
return ctrl.Result{}, nil
}
} else {
// A registration-only runner does not exist and is needed, hence create it.
runnerForScaleFromToZero, err := r.newRunner(rs)
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to create runner for scale from/to zero: %v", err)
}
runnerForScaleFromToZero.ObjectMeta.Name = registrationOnlyRunnerNsName.Name
runnerForScaleFromToZero.ObjectMeta.GenerateName = ""
runnerForScaleFromToZero.ObjectMeta.Labels = nil
metav1.SetMetaDataAnnotation(&runnerForScaleFromToZero.ObjectMeta, annotationKeyRegistrationOnly, "true")
if err := r.Client.Create(ctx, &runnerForScaleFromToZero); err != nil {
log.Error(err, "Failed to create runner for scale from/to zero")
return ctrl.Result{}, err
}
// We can continue to deleting runner pods only after the
// registration-only runner gets registered.
return ctrl.Result{}, nil
}
} else {
// A registration-only runner exists and is not needed, hence delete it.
if registrationOnlyRunnerExists {
if err := r.Client.Delete(ctx, &registrationOnlyRunner); err != nil {
log.Error(err, "Retrying soon because we failed to delete registration-only runner")
return ctrl.Result{Requeue: true}, nil
}
}
}
if available > desired {
n := available - desired
@@ -244,6 +250,8 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
n = len(deletionCandidates)
}
log.V(0).Info(fmt.Sprintf("Deleting %d runner(s)", n), "desired", desired, "available", available, "ready", ready)
for i := 0; i < n; i++ {
if err := r.Client.Delete(ctx, &deletionCandidates[i]); client.IgnoreNotFound(err) != nil {
log.Error(err, "Failed to delete runner resource")