chore: bump chart to latest (#1300)

* chore: bump chart to latest

Bumps the chart version along with the controller version.
We bump the patch number for the chart as the release for the controller is a patch release.
That's the same handling as we've done in the previous version ecc8b4472a

As always, be sure to upgrade CRDs before updating the controller version!
Otherwise it can break in interesting ways.

* docs: expand on CRD upgrade requirement

Co-authored-by: Callum Tait <15716903+toast-gear@users.noreply.github.com>

.github/workflows/on-push-master-publish-chart.yml

@@ -114,7 +114,7 @@ jobs:
           git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
 
       - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.3.0
+        uses: helm/chart-releaser-action@v1.4.0
         env:
           CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
 

.github/workflows/runners.yml

@@ -15,7 +15,7 @@ on:
       - '!**.md'
 
 env:
-  RUNNER_VERSION: 2.289.1
+  RUNNER_VERSION: 2.289.2
   DOCKER_VERSION: 20.10.12
   DOCKERHUB_USERNAME: summerwind
 

README.md

@@ -233,9 +233,9 @@ If you plan on installing all instances of the controller stack into a single na
 1. All resources per stack must have a unique, in the case of Helm this can be done by giving each install a unique release name, or via the `fullnameOverride` properties.
 2. `authSecret.name` needs be unique per stack when each stack is tied to runners in different GitHub organizations and repositories AND you want your GitHub credentials to narrowly scoped.
 3. `leaderElectionId` needs to be unique per stack. If this is not unique to the stack the controller tries to race onto the leader election lock resulting in only one stack working concurrently. Your controller will be stuck with a log message something like this `attempting to acquire leader lease arc-controllers/actions-runner-controller...`
-4. The MutatingWebhookConfiguration in each stack must include a namespace selector for that stacks corresponding runners namespace, this is already configured in the helm chart.
+4. The MutatingWebhookConfiguration in each stack must include a namespace selector for that stacks corresponding runner namespace, this is already configured in the helm chart.
 
-Alternatively, you can install each controller stack into a unique namespace (relative to other controller stacks in the cluster), avoiding these potential pitfalls.
+Alternatively, you can install each controller stack into a unique namespace (relative to other controller stacks in the cluster). Implementing ARC this way avoids the first, second and third pitfalls (you still need to set the corresponding namespace selector for each stacks mutating webhook)
 
 ## Usage
 
@@ -668,7 +668,7 @@ The primary benefit of autoscaling on Webhook compared to the pull driven scalin
 
 > You can learn the implementation details in [#282](https://github.com/actions-runner-controller/actions-runner-controller/pull/282)
 
-To enable this feature, you firstly need to install the webhook server, currently, only our Helm chart has the ability install it:
+To enable this feature, you first need to install the GitHub webhook server. To install via our Helm chart,
 _[see the values documentation for all configuration options](https://github.com/actions-runner-controller/actions-runner-controller/blob/master/charts/actions-runner-controller/README.md)_
 
 ```console

api/v1alpha1/runner_types.go

@@ -181,6 +181,9 @@ func (rs *RunnerSpec) ValidateRepository() error {
 
 // RunnerStatus defines the observed state of Runner
 type RunnerStatus struct {
+	// Turns true only if the runner pod is ready.
+	// +optional
+	Ready bool `json:"ready"`
 	// +optional
 	Registration RunnerStatusRegistration `json:"registration"`
 	// +optional

charts/actions-runner-controller/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.17.1
+version: 0.17.2
 
 # Used as the default manager tag value when no tag property is provided in the values.yaml
-appVersion: 0.22.1
+appVersion: 0.22.2
 
 home: https://github.com/actions-runner-controller/actions-runner-controller
 

charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml

@@ -5126,6 +5126,9 @@ spec:
                   type: string
                 phase:
                   type: string
+                ready:
+                  description: Turns true only if the runner pod is ready.
+                  type: boolean
                 reason:
                   type: string
                 registration:

charts/actions-runner-controller/docs/UPGRADING.md

@@ -18,7 +18,7 @@ Due to the above you can't just do a `helm upgrade` to release the latest versio
 
 ## Steps
 
-1. Upgrade CRDs
+1. Upgrade CRDs, this isn't optional, the CRDs you are using must be those that correspond with the version of the controller you are installing
 
 ```shell
 # REMEMBER TO UPDATE THE CHART_VERSION TO RELEVANT CHART VERISON!!!!

config/crd/bases/actions.summerwind.dev_runners.yaml

@@ -5126,6 +5126,9 @@ spec:
                   type: string
                 phase:
                   type: string
+                ready:
+                  description: Turns true only if the runner pod is ready.
+                  type: boolean
                 reason:
                   type: string
                 registration:

config/default/gh-webhook-server-auth-proxy-patch.yaml

@@ -0,0 +1,23 @@
+# This patch injects an HTTP proxy sidecar container that performs RBAC
+# authorization against the Kubernetes API using SubjectAccessReviews.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: github-webhook-server
+spec:
+  template:
+    spec:
+      containers:
+        - name: kube-rbac-proxy
+          image: quay.io/brancz/kube-rbac-proxy:v0.10.0
+          args:
+            - '--secure-listen-address=0.0.0.0:8443'
+            - '--upstream=http://127.0.0.1:8080/'
+            - '--logtostderr=true'
+            - '--v=10'
+          ports:
+            - containerPort: 8443
+              name: https
+        - name: github-webhook-server
+          args:
+            - '--metrics-addr=127.0.0.1:8080'

config/default/kustomization.yaml

@@ -20,19 +20,22 @@ bases:
 - ../webhook
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 - ../certmanager
-# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 
+# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
+# [GH_WEBHOOK_SERVER] To enable the GitHub webhook server, uncomment all sections with 'GH_WEBHOOK_SERVER'.
+#- ../github-webhook-server
 
 patchesStrategicMerge:
-  # Protect the /metrics endpoint by putting it behind auth.
-  # Only one of manager_auth_proxy_patch.yaml and
-  # manager_prometheus_metrics_patch.yaml should be enabled.
+# Protect the /metrics endpoint by putting it behind auth.
+# Only one of manager_auth_proxy_patch.yaml and
+# manager_prometheus_metrics_patch.yaml should be enabled.
 - manager_auth_proxy_patch.yaml
-  # If you want your controller-manager to expose the /metrics
-  # endpoint w/o any authn/z, uncomment the following line and
-  # comment manager_auth_proxy_patch.yaml.
-  # Only one of manager_auth_proxy_patch.yaml and
-  # manager_prometheus_metrics_patch.yaml should be enabled.
+
+# If you want your controller-manager to expose the /metrics
+# endpoint w/o any authn/z, uncomment the following line and
+# comment manager_auth_proxy_patch.yaml.
+# Only one of manager_auth_proxy_patch.yaml and
+# manager_prometheus_metrics_patch.yaml should be enabled.
 #- manager_prometheus_metrics_patch.yaml
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml
@@ -43,6 +46,10 @@ patchesStrategicMerge:
 # 'CERTMANAGER' needs to be enabled to use ca injection
 - webhookcainjection_patch.yaml
 
+# [GH_WEBHOOK_SERVER] To enable the GitHub webhook server, uncomment all sections with 'GH_WEBHOOK_SERVER'.
+# Protect the GitHub webhook server metrics endpoint by putting it behind auth.
+# - gh-webhook-server-auth-proxy-patch.yaml
+
 # the following config is for teaching kustomize how to do var substitution
 vars:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.

config/default/manager_auth_proxy_patch.yaml

@@ -23,4 +23,3 @@ spec:
         args:
         - "--metrics-addr=127.0.0.1:8080"
         - "--enable-leader-election"
-        - "--sync-period=10m"

config/github-webhook-server/deployment.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/component: github-webhook-server
+    app.kubernetes.io/part-of: actions-runner-controller
+  name: github-webhook-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: github-webhook-server
+      app.kubernetes.io/part-of: actions-runner-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: github-webhook-server
+        app.kubernetes.io/part-of: actions-runner-controller
+    spec:
+      containers:
+        - name: github-webhook-server
+          image: controller:latest
+          command:
+            - '/github-webhook-server'
+          env:
+            - name: GITHUB_WEBHOOK_SECRET_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  key: github_webhook_secret_token
+                  name: github-webhook-server
+                  optional: true
+          ports:
+            - containerPort: 8000
+              name: http
+              protocol: TCP
+      serviceAccountName: github-webhook-server
+      terminationGracePeriodSeconds: 10

config/github-webhook-server/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+images:
+  - name: controller
+    newName: summerwind/actions-runner-controller
+    newTag: latest
+
+resources:
+  - deployment.yaml
+  - rbac.yaml
+  - service.yaml

config/github-webhook-server/rbac.yaml

@@ -0,0 +1,113 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: github-webhook-server
+    app.kubernetes.io/part-of: actions-runner-controller
+  name: github-webhook-server
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/component: github-webhook-server
+    app.kubernetes.io/part-of: actions-runner-controller
+  name: github-webhook-server
+rules:
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - horizontalrunnerautoscalers
+    verbs:
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - horizontalrunnerautoscalers/finalizers
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - horizontalrunnerautoscalers/status
+    verbs:
+      - get
+      - patch
+      - update
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - runnersets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - runnerdeployments
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - runnerdeployments/finalizers
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - runnerdeployments/status
+    verbs:
+      - get
+      - patch
+      - update
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: github-webhook-server
+    app.kubernetes.io/part-of: actions-runner-controller
+  name: github-webhook-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: github-webhook-server
+subjects:
+  - kind: ServiceAccount
+    name: github-webhook-server

config/github-webhook-server/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: github-webhook-server
+    app.kubernetes.io/part-of: actions-runner-controller
+  name: github-webhook-server
+spec:
+  ports:
+    - port: 80
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app.kubernetes.io/component: github-webhook-server
+    app.kubernetes.io/part-of: actions-runner-controller

controllers/runner_controller.go

@@ -132,7 +132,9 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 		phase = "Created"
 	}
 
-	if runner.Status.Phase != phase {
+	ready := runnerPodReady(&pod)
+
+	if runner.Status.Phase != phase || runner.Status.Ready != ready {
 		if pod.Status.Phase == corev1.PodRunning {
 			// Seeing this message, you can expect the runner to become `Running` soon.
 			log.V(1).Info(
@@ -143,6 +145,7 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 
 		updated := runner.DeepCopy()
 		updated.Status.Phase = phase
+		updated.Status.Ready = ready
 		updated.Status.Reason = pod.Status.Reason
 		updated.Status.Message = pod.Status.Message
 
@@ -155,6 +158,18 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 	return ctrl.Result{}, nil
 }
 
+func runnerPodReady(pod *corev1.Pod) bool {
+	for _, c := range pod.Status.Conditions {
+		if c.Type != corev1.PodReady {
+			continue
+		}
+
+		return c.Status == corev1.ConditionTrue
+	}
+
+	return false
+}
+
 func runnerContainerExitCode(pod *corev1.Pod) *int32 {
 	for _, status := range pod.Status.ContainerStatuses {
 		if status.Name != containerName {

github/github.go

@@ -153,8 +153,18 @@ func (c *Client) GetRegistrationToken(ctx context.Context, enterprise, org, repo
 	key := getRegistrationKey(org, repo, enterprise)
 	rt, ok := c.regTokens[key]
 
-	// we like to give runners a chance that are just starting up and may miss the expiration date by a bit
-	runnerStartupTimeout := 3 * time.Minute
+	// We'd like to allow the runner just starting up to miss the expiration date by a bit.
+	// Note that this means that we're going to cache Creation Registraion Token API response longer than the
+	// recommended cache duration.
+	//
+	// https://docs.github.com/en/rest/reference/actions#create-a-registration-token-for-a-repository
+	// https://docs.github.com/en/rest/reference/actions#create-a-registration-token-for-an-organization
+	// https://docs.github.com/en/rest/reference/actions#create-a-registration-token-for-an-enterprise
+	// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#conditional-requests
+	//
+	// This is currently set to 30 minutes as the result of the discussion took place at the following issue:
+	// https://github.com/actions-runner-controller/actions-runner-controller/issues/1295
+	runnerStartupTimeout := 30 * time.Minute
 
 	if ok && rt.GetExpiresAt().After(time.Now().Add(runnerStartupTimeout)) {
 		return rt, nil

runner/Dockerfile

@@ -83,7 +83,7 @@ ENV HOME=/home/runner
 #
 # If you're willing to uncomment the following line, you'd also need to comment-out the
 #   && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${ARCH}-${RUNNER_VERSION}.tar.gz \
-# line in the next `RUN` command in this Dockerfile, to avoid overwiding this runner.tar.gz with a remote one.
+# line in the next `RUN` command in this Dockerfile, to avoid overwiting this runner.tar.gz with a remote one.
 
 # COPY actions-runner-linux-x64-2.280.3.tar.gz /runnertmp/runner.tar.gz
 

runner/entrypoint.sh

@@ -151,7 +151,7 @@ cat .runner
 #     https://api.github.com/repos/USER/REPO/actions/runners/171
 
 if [ -z "${UNITTEST:-}" ]; then
-  mkdir ./externals
+  mkdir -p ./externals
   # Hack due to the DinD volumes
   mv ./externalstmp/* ./externals/
 fi

test/e2e/e2e_test.go

@@ -37,13 +37,23 @@ var (
 		},
 		{
 			Dockerfile: "../../runner/Dockerfile",
-			Args:       []testing.BuildArg{},
-			Image:      runnerImage,
+			Args: []testing.BuildArg{
+				{
+					Name:  "RUNNER_VERSION",
+					Value: "2.289.2",
+				},
+			},
+			Image: runnerImage,
 		},
 		{
 			Dockerfile: "../../runner/Dockerfile.dindrunner",
-			Args:       []testing.BuildArg{},
-			Image:      runnerDindImage,
+			Args: []testing.BuildArg{
+				{
+					Name:  "RUNNER_VERSION",
+					Value: "2.289.2",
+				},
+			},
+			Image: runnerDindImage,
 		},
 	}
 
@@ -58,7 +68,7 @@ var (
 	}
 
 	commonScriptEnv = []string{
-		"SYNC_PERIOD=" + "10s",
+		"SYNC_PERIOD=" + "30m",
 		"NAME=" + controllerImageRepo,
 		"VERSION=" + controllerImageTag,
 		"RUNNER_TAG=" + runnerImageTag,