RunnerSet: Automatic-recovery from registration timeout and deregistration on pod termination (#652)

Ref #629
Ref #613
Ref #612

acceptance/testdata/repo.runnerset.hra.yaml

@@ -16,6 +16,8 @@ spec:
   # RunnerSet doesn't support scale from/to zero yet
   minReplicas: 1
   maxReplicas: 5
+  # This should be less than 600(seconds, the default) for faster testing
+  scaleDownDelaySecondsAfterScaleOut: 60
   metrics:
   - type: PercentageRunnersBusy
     scaleUpThreshold: '0.75'

acceptance/testdata/repo.runnerset.yaml

@@ -14,6 +14,10 @@ spec:
   serviceName: example-runnerset
 
   #replicas: 1
+
+  # From my limited testing, `ephemeral: true` is more reliable.
+  # Seomtimes, updating already deployed runners from `ephemeral: false` to `ephemeral: true` seems to
+  # result in queued jobs hanging forever.
   ephemeral: false
 
   repository: ${TEST_REPO}