mirror of
https://github.com/actions/actions-runner-controller.git
synced 2025-12-10 11:41:27 +00:00
Fix rate limit and runner registration logic (#309)
* errors.Is compares all members of a struct to return true which never happened * switched to type check instead of exact value check * notRegistered was using double negation in if statement which lead to unregistering runners after the registration timeout
This commit is contained in:
@@ -244,16 +244,18 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
|
|||||||
return ctrl.Result{}, err
|
return ctrl.Result{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var notRegistered bool
|
notRegistered := false
|
||||||
|
|
||||||
runnerBusy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
|
runnerBusy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, github.RunnerNotFound{}) {
|
var e *github.RunnerNotFound
|
||||||
|
if errors.As(err, &e) {
|
||||||
log.Error(err, "Failed to check if runner is busy. Probably this runner has never been successfully registered to GitHub.")
|
log.Error(err, "Failed to check if runner is busy. Probably this runner has never been successfully registered to GitHub.")
|
||||||
|
|
||||||
notRegistered = true
|
notRegistered = true
|
||||||
} else {
|
} else {
|
||||||
if errors.Is(err, &gogithub.RateLimitError{}) {
|
var e *gogithub.RateLimitError
|
||||||
|
if errors.As(err, &e) {
|
||||||
// We log the underlying error when we failed calling GitHub API to list or unregisters,
|
// We log the underlying error when we failed calling GitHub API to list or unregisters,
|
||||||
// or the runner is still busy.
|
// or the runner is still busy.
|
||||||
log.Error(
|
log.Error(
|
||||||
@@ -284,7 +286,7 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
|
|||||||
currentTime := time.Now()
|
currentTime := time.Now()
|
||||||
registrationDidTimeout := currentTime.Sub(pod.CreationTimestamp.Add(registrationTimeout)) > 0
|
registrationDidTimeout := currentTime.Sub(pod.CreationTimestamp.Add(registrationTimeout)) > 0
|
||||||
|
|
||||||
if !notRegistered && registrationDidTimeout {
|
if notRegistered && registrationDidTimeout {
|
||||||
log.Info(
|
log.Info(
|
||||||
"Runner failed to register itself to GitHub in timely manner. "+
|
"Runner failed to register itself to GitHub in timely manner. "+
|
||||||
"Recreating the pod to see if it resolves the issue. "+
|
"Recreating the pod to see if it resolves the issue. "+
|
||||||
|
|||||||
@@ -107,39 +107,36 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
|
|||||||
for _, runner := range myRunners {
|
for _, runner := range myRunners {
|
||||||
busy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
|
busy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(err, "Failed to check if runner is busy. Probably this runner has never been successfully registered to GitHub, and therefore we prioritize it for deletion", "runnerName", runner.Name)
|
notRegistered := false
|
||||||
|
|
||||||
var notRegistered bool
|
var e *github.RunnerNotFound
|
||||||
|
if errors.As(err, &e) {
|
||||||
|
log.Error(err, "Failed to check if runner is busy. Probably this runner has never been successfully registered to GitHub, and therefore we prioritize it for deletion", "runnerName", runner.Name)
|
||||||
|
notRegistered = true
|
||||||
|
} else {
|
||||||
|
var e *gogithub.RateLimitError
|
||||||
|
if errors.As(err, &e) {
|
||||||
|
// We log the underlying error when we failed calling GitHub API to list or unregisters,
|
||||||
|
// or the runner is still busy.
|
||||||
|
log.Error(
|
||||||
|
err,
|
||||||
|
fmt.Sprintf(
|
||||||
|
"Failed to check if runner is busy due to GitHub API rate limit. Retrying in %s to avoid excessive GitHub API calls",
|
||||||
|
retryDelayOnGitHubAPIRateLimitError,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
if err != nil {
|
return ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
|
||||||
if errors.Is(err, github.RunnerNotFound{}) {
|
|
||||||
log.Error(err, "Failed to check if runner is busy. Probably this runner has never been successfully registered to GitHub.")
|
|
||||||
|
|
||||||
notRegistered = true
|
|
||||||
} else {
|
|
||||||
if errors.Is(err, &gogithub.RateLimitError{}) {
|
|
||||||
// We log the underlying error when we failed calling GitHub API to list or unregisters,
|
|
||||||
// or the runner is still busy.
|
|
||||||
log.Error(
|
|
||||||
err,
|
|
||||||
fmt.Sprintf(
|
|
||||||
"Failed to check if runner is busy due to GitHub API rate limit. Retrying in %s to avoid excessive GitHub API calls",
|
|
||||||
retryDelayOnGitHubAPIRateLimitError,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
return ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return ctrl.Result{}, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return ctrl.Result{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
registrationTimeout := 15 * time.Minute
|
registrationTimeout := 15 * time.Minute
|
||||||
currentTime := time.Now()
|
currentTime := time.Now()
|
||||||
registrationDidTimeout := currentTime.Sub(runner.CreationTimestamp.Add(registrationTimeout)) > 0
|
registrationDidTimeout := currentTime.Sub(runner.CreationTimestamp.Add(registrationTimeout)) > 0
|
||||||
|
|
||||||
if !notRegistered && registrationDidTimeout {
|
if notRegistered && registrationDidTimeout {
|
||||||
log.Info(
|
log.Info(
|
||||||
"Runner failed to register itself to GitHub in timely manner. "+
|
"Runner failed to register itself to GitHub in timely manner. "+
|
||||||
"Recreating the pod to see if it resolves the issue. "+
|
"Recreating the pod to see if it resolves the issue. "+
|
||||||
|
|||||||
@@ -287,7 +287,7 @@ type RunnerNotFound struct {
|
|||||||
runnerName string
|
runnerName string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e RunnerNotFound) Error() string {
|
func (e *RunnerNotFound) Error() string {
|
||||||
return fmt.Sprintf("runner %q not found", e.runnerName)
|
return fmt.Sprintf("runner %q not found", e.runnerName)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -303,5 +303,5 @@ func (r *Client) IsRunnerBusy(ctx context.Context, enterprise, org, repo, name s
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return false, RunnerNotFound{runnerName: name}
|
return false, &RunnerNotFound{runnerName: name}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user