chart: Bump chart and app versions for ARC 0.27.0 (#2160)

* Update controller package names to match the owning API group name

* feedback.

Co-authored-by: Bassem Dghaidi <568794+Link-@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,8 +1,18 @@
 name: Bug Report
 description: File a bug report
-title: "Bug"
-labels: ["bug"]
+title: "<Please write what didn't work for you here>"
+labels: ["bug", "needs triage"]
 body:
+- type: checkboxes
+  id: read-troubleshooting-guide
+  attributes:
+    label: Checks
+    description: Please check all the boxes below before submitting
+    options:
+    - label: I've already read https://github.com/actions/actions-runner-controller/blob/master/TROUBLESHOOTING.md and I'm sure my issue is not covered in the troubleshooting guide.
+      required: true
+    - label: I'm not using a custom entrypoint in my runner image
+      required: true
 - type: input
   id: controller-version
   attributes:
@@ -41,7 +51,7 @@ body:
     label: cert-manager installation
     description: Confirm that you've installed cert-manager correctly by answering a few questions
     placeholder: |
-      - Did you follow https://github.com/actions-runner-controller/actions-runner-controller#installation? If not, describe the installation process so that we can reproduce your environment.
+      - Did you follow https://github.com/actions/actions-runner-controller#installation? If not, describe the installation process so that we can reproduce your environment.
       - Are you sure you've installed cert-manager from an official source?
       (Note that we won't provide user support for cert-manager itself. Make sure cert-manager is fully working before testing ARC or reporting a bug
   validations:
@@ -50,16 +60,18 @@ body:
   id: checks
   attributes:
     label: Checks
-    description: Please check the boxes below before submitting
+    description: Please check all the boxes below before submitting
     options:
-    - label: This isn't a question or user support case (For Q&A and community support, go to [Discussions](https://github.com/actions-runner-controller/actions-runner-controller/discussions). It might also be a good idea to contract with any of contributors and maintainers if your business is so critical and therefore you need priority support
+    - label: This isn't a question or user support case (For Q&A and community support, go to [Discussions](https://github.com/actions/actions-runner-controller/discussions). It might also be a good idea to contract with any of contributors and maintainers if your business is so critical and therefore you need priority support
       required: true
-    - label: I've read [releasenotes](https://github.com/actions-runner-controller/actions-runner-controller/tree/master/docs/releasenotes) before submitting this issue and I'm sure it's not due to any recently-introduced backward-incompatible changes
+    - label: I've read [releasenotes](https://github.com/actions/actions-runner-controller/tree/master/docs/releasenotes) before submitting this issue and I'm sure it's not due to any recently-introduced backward-incompatible changes
       required: true
     - label: My actions-runner-controller version (v0.x.y) does support the feature
       required: true
     - label: I've already upgraded ARC (including the CRDs, see charts/actions-runner-controller/docs/UPGRADING.md for details) to the latest and it didn't fix the issue
       required: true
+    - label: I've migrated to the workflow job webhook event (if you using webhook driven scaling)
+      required: true
 - type: textarea
   id: resource-definitions
   attributes:
@@ -129,10 +141,12 @@ body:
 - type: textarea
   id: controller-logs
   attributes:
-    label: Controller Logs
-    description: "NEVER EVER OMIT THIS! Include logs from `actions-runner-controller`'s controller-manager pod"
+    label: Whole Controller Logs
+    description: "NEVER EVER OMIT THIS! Include logs from `actions-runner-controller`'s controller-manager pod. Don't omit the parts you think irrelevant!"
     render: shell
     placeholder: |
+      PROVIDE THE LOGS VIA A GIST LINK (https://gist.github.com/), NOT DIRECTLY IN THIS TEXT AREA
+      
       To grab controller logs:
 
       # Set NS according to your setup
@@ -142,17 +156,17 @@ body:
       kubectl -n $NS get po
 
       kubectl -n $NS logs $POD_NAME > arc.log
-
-      Upload it to e.g. https://gist.github.com/ and paste the link to it here.
   validations:
     required: true
 - type: textarea
   id: runner-pod-logs
   attributes:
-    label: Runner Pod Logs
-    description: "Include logs from runner pod(s)"
+    label: Whole Runner Pod Logs
+    description: "Include logs from runner pod(s). Please don't omit the parts you think irrelevant!"
     render: shell
     placeholder: |
+      PROVIDE THE WHOLE LOGS VIA A GIST LINK (https://gist.github.com/), NOT DIRECTLY IN THIS TEXT AREA
+      
       To grab the runner pod logs:
 
       # Set NS according to your setup. It should match your RunnerDeployment's metadata.namespace.
@@ -163,8 +177,8 @@ body:
 
       kubectl -n $NS logs $POD_NAME -c runner > runnerpod_runner.log
       kubectl -n $NS logs $POD_NAME -c docker > runnerpod_docker.log
-
-      Upload it to e.g. https://gist.github.com/ and paste the link to it here.
+      
+      If any of the containers are getting terminated immediately, try adding `--previous` to the kubectl-logs command to obtain logs emitted before the termination.
   validations:
     required: true
 - type: textarea

.github/ISSUE_TEMPLATE/config.yml

@@ -1,15 +1,14 @@
-# Blank issues are mainly for maintainers who are known to write complete issue descriptions without need to following a form
-blank_issues_enabled: true
+blank_issues_enabled: false
 contact_links:
 - name: Sponsor ARC Maintainers
   about: If your business relies on the continued maintainance of actions-runner-controller, please consider sponsoring the project and the maintainers.
-  url: https://github.com/actions-runner-controller/actions-runner-controller/tree/master/CODEOWNERS
+  url: https://github.com/actions/actions-runner-controller/tree/master/CODEOWNERS
 - name: Ideas and Feature Requests
   about: Wanna request a feature? Create a discussion and collect :+1:s first.
-  url: https://github.com/actions-runner-controller/actions-runner-controller/discussions/new?category=ideas
+  url: https://github.com/actions/actions-runner-controller/discussions/new?category=ideas
 - name: Questions and User Support
   about: Need support using ARC? We use Discussions as the place to provide community support.
-  url: https://github.com/actions-runner-controller/actions-runner-controller/discussions/new?category=questions
+  url: https://github.com/actions/actions-runner-controller/discussions/new?category=questions
 - name: Need Paid Support?
   about: Consider contracting with any of the actions-runner-controller maintainers and contributors.
-  url: https://github.com/actions-runner-controller/actions-runner-controller/tree/master/CODEOWNERS
+  url: https://github.com/actions/actions-runner-controller/tree/master/CODEOWNERS

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,19 +1,21 @@
 ---
 name: Feature request
 about: Suggest an idea for this project
+labels: ["enhancement", "needs triage"]
 title: ''
 assignees: ''
-
 ---
 
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+### What would you like added?
 
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
+*A clear and concise description of what you want to happen.*
 
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
+Note: Feature requests to integrate vendor specific cloud tools (e.g. `awscli`, `gcloud-sdk`, `azure-cli`) will likely be rejected as the Runner image aims to be vendor agnostic.
 
-**Additional context**
-Add any other context or screenshots about the feature request here.
+### Why is this needed?
+
+*A clear and concise description of any alternative solutions or features you've considered.*
+
+### Additional context
+
+*Add any other context or screenshots about the feature request here.*

.github/actions/setup-docker-environment/action.yaml

@@ -14,18 +14,13 @@ inputs:
     description: "GHCR password. Usually set from the secrets.GITHUB_TOKEN variable"
     required: true
 
-outputs:
-  sha_short:
-    description: "The short SHA used for image builds"
-    value: ${{ steps.vars.outputs.sha_short }}
-
 runs:
   using: "composite"
   steps:
     - name: Get Short SHA
       id: vars
       run: |
-        echo ::set-output name=sha_short::${GITHUB_SHA::7}
+        echo "sha_short=${GITHUB_SHA::7}" >> $GITHUB_ENV
       shell: bash
 
     - name: Set up QEMU
@@ -37,12 +32,14 @@ runs:
         version: latest
 
     - name: Login to DockerHub
+      if: ${{ github.event_name == 'release' || github.event_name == 'push' && github.ref == 'refs/heads/master' && inputs.password != ''  }}
       uses: docker/login-action@v2
       with:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
 
     - name: Login to GitHub Container Registry
+      if: ${{ github.event_name == 'release' || github.event_name == 'push' && github.ref == 'refs/heads/master' && inputs.ghcr_password != ''  }}
       uses: docker/login-action@v2
       with:
         registry: ghcr.io

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "gomod" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/renovate.json5

@@ -30,8 +30,10 @@
     },
     {
       "fileMatch": [
-        "runner/actions-runner.dockerfile",
-        "runner/actions-runner-dind.dockerfile"
+        "runner/actions-runner.ubuntu-20.04.dockerfile",
+        "runner/actions-runner.ubuntu-22.04.dockerfile",
+        "runner/actions-runner-dind.ubuntu-20.04.dockerfile",
+        "runner/actions-runner-dind-rootless.ubuntu-20.04.dockerfile"
       ],
       "matchStrings": ["RUNNER_VERSION=+(?<currentValue>.*?)\\n"],
       "depNameTemplate": "actions/runner",

.github/workflows/golangci-lint.yaml

@@ -0,0 +1,23 @@
+name: golangci-lint
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+permissions:
+  contents: read
+  pull-requests: read
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+      - uses: actions/checkout@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          only-new-issues: true
+          version: v1.49.0

.github/workflows/publish-arc.yaml

@@ -1,16 +1,34 @@
 name: Publish ARC
 
+# Revert to https://github.com/actions-runner-controller/releases#releases
+# for details on why we use this approach
 on:
   release:
     types:
       - published
+  workflow_dispatch:
+    inputs:
+      release_tag_name:
+        description: 'Tag name of the release to publish'
+        required: true
+      push_to_registries:
+        description: 'Push images to registries'
+        required: true
+        type: boolean
+        default: false
+
+permissions:
+ contents: write 
+ packages: write
+
+env:
+  TARGET_ORG: actions-runner-controller
+  TARGET_REPO: actions-runner-controller
 
 jobs:
   release-controller:
     name: Release
     runs-on: ubuntu-latest
-    env:
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USER }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -30,8 +48,14 @@ jobs:
           tar zxvf ghr_v0.13.0_linux_amd64.tar.gz
           sudo mv ghr_v0.13.0_linux_amd64/ghr /usr/local/bin
 
-      - name: Set version
-        run: echo "VERSION=$(cat ${GITHUB_EVENT_PATH} | jq -r '.release.tag_name')" >> $GITHUB_ENV
+      - name: Set version env variable
+        run: |
+          # Define the release tag name based on the event type
+          if [[ "${{ github.event_name }}" == "release" ]]; then
+            echo "VERSION=$(cat ${GITHUB_EVENT_PATH} | jq -r '.release.tag_name')" >> $GITHUB_ENV
+          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "VERSION=${{ inputs.release_tag_name }}" >> $GITHUB_ENV
+          fi
 
       - name: Upload artifacts
         env:
@@ -39,25 +63,39 @@ jobs:
         run: |
           make github-release
 
-      - name: Setup Docker Environment
-        id: vars
-        uses: ./.github/actions/setup-docker-environment
+      - name: Get Token
+        id: get_workflow_token
+        uses: peter-murray/workflow-application-token-action@8e1ba3bf1619726336414f1014e37f17fbadf1db
         with:
-          username: ${{ env.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
-          ghcr_username: ${{ github.actor }}
-          ghcr_password: ${{ secrets.GITHUB_TOKEN }}
+          application_id: ${{ secrets.ACTIONS_ACCESS_APP_ID }}
+          application_private_key: ${{ secrets.ACTIONS_ACCESS_PK }}
+          organization: ${{ env.TARGET_ORG }}
 
-      - name: Build and Push
-        uses: docker/build-push-action@v3
-        with:
-          file: Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: |
-            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:latest
-            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}
-            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}-${{ steps.vars.outputs.sha_short }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+      - name: Resolve push to registries
+        run: |
+          # Define the push to registries based on the event type
+          if [[ "${{ github.event_name }}" == "release" ]]; then
+            echo "PUSH_TO_REGISTRIES=true" >> $GITHUB_ENV
+          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "PUSH_TO_REGISTRIES=${{ inputs.push_to_registries }}" >> $GITHUB_ENV
+          fi
 
+      - name: Trigger Build And Push Images To Registries
+        run: |
+          # Authenticate
+          gh auth login --with-token <<< ${{ steps.get_workflow_token.outputs.token }}
+
+          # Trigger the workflow run
+          jq -n '{"event_type": "arc", "client_payload": {"release_tag_name": "${{ env.VERSION }}", "push_to_registries": "${{ env.PUSH_TO_REGISTRIES }}" }}' \
+            | gh api -X POST /repos/actions-runner-controller/releases/dispatches --input -
+
+      - name: Job summary
+        run: |
+          echo "The [publish-arc](https://github.com/actions-runner-controller/releases/blob/main/.github/workflows/publish-arc.yaml) workflow has been triggered!" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Parameters:**" >> $GITHUB_STEP_SUMMARY
+          echo "- Release tag: ${{ env.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Push to registries: ${{ env.PUSH_TO_REGISTRIES }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Status:**" >> $GITHUB_STEP_SUMMARY
+          echo "[https://github.com/actions-runner-controller/releases/actions/workflows/publish-arc.yaml](https://github.com/actions-runner-controller/releases/actions/workflows/publish-arc.yaml)" >> $GITHUB_STEP_SUMMARY

.github/workflows/publish-canary.yaml

@@ -1,5 +1,7 @@
 name: Publish Canary Image
 
+# Revert to https://github.com/actions-runner-controller/releases#releases
+# for details on why we use this approach
 on:
   push:
     branches:
@@ -19,40 +21,50 @@ on:
       - 'LICENSE'
       - 'Makefile'
 
+env:
+  # Safeguard to prevent pushing images to registeries after build
+  PUSH_TO_REGISTRIES: true
+  TARGET_ORG: actions-runner-controller
+  TARGET_REPO: actions-runner-controller
+
 # https://docs.github.com/en/rest/overview/permissions-required-for-github-apps
 permissions:
   contents: read
-  packages: write  
 
 jobs:
   canary-build:
-    name: Build and Publish Canary Image  
+    name: Build and Publish Canary Image
     runs-on: ubuntu-latest
     env:
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USER }}
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Setup Docker Environment
-        id: vars
-        uses: ./.github/actions/setup-docker-environment
+      - name: Get Token
+        id: get_workflow_token
+        uses: peter-murray/workflow-application-token-action@8e1ba3bf1619726336414f1014e37f17fbadf1db
         with:
-          username: ${{ env.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
-          ghcr_username: ${{ github.actor }}
-          ghcr_password: ${{ secrets.GITHUB_TOKEN }}
+          application_id: ${{ secrets.ACTIONS_ACCESS_APP_ID }}
+          application_private_key: ${{ secrets.ACTIONS_ACCESS_PK }}
+          organization: ${{ env.TARGET_ORG }}
 
-      # Considered unstable builds
-      # See Issue #285, PR #286, and PR #323 for more information
-      - name: Build and Push
-        uses: docker/build-push-action@v3
-        with:
-          file: Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: |
-            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:canary
-            ghcr.io/actions-runner-controller/actions-runner-controller:canary
-          cache-from: type=gha,scope=arc-canary
-          cache-to: type=gha,mode=max,scope=arc-canary
+      - name: Trigger Build And Push Images To Registries
+        run: |
+          # Authenticate
+          gh auth login --with-token <<< ${{ steps.get_workflow_token.outputs.token }}
+
+          # Trigger the workflow run
+          jq -n '{"event_type": "canary", "client_payload": {"sha": "${{ github.sha }}", "push_to_registries": ${{ env.PUSH_TO_REGISTRIES }}}}' \
+            | gh api -X POST /repos/actions-runner-controller/releases/dispatches --input -
+
+      - name: Job summary
+        run: |
+          echo "The [publish-canary](https://github.com/actions-runner-controller/releases/blob/main/.github/workflows/publish-canary.yaml) workflow has been triggered!" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Parameters:**" >> $GITHUB_STEP_SUMMARY
+          echo "- sha: ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Push to registries: ${{ env.PUSH_TO_REGISTRIES }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Status:**" >> $GITHUB_STEP_SUMMARY
+          echo "[https://github.com/actions-runner-controller/releases/actions/workflows/publish-canary.yaml](https://github.com/actions-runner-controller/releases/actions/workflows/publish-canary.yaml)" >> $GITHUB_STEP_SUMMARY

.github/workflows/publish-chart.yaml

@@ -1,5 +1,7 @@
 name: Publish Helm Chart
 
+# Revert to https://github.com/actions-runner-controller/releases#releases
+# for details on why we use this approach
 on:
   push:
     branches:
@@ -31,7 +33,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v2.1
+        uses: azure/setup-helm@v3.4
         with:
           version: ${{ env.HELM_VERSION }}
 
@@ -57,7 +59,7 @@ jobs:
           python-version: '3.7'
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.2.1
+        uses: helm/chart-testing-action@v2.3.1
 
       - name: Run chart-testing (list-changed)
         id: list-changed
@@ -73,7 +75,7 @@ jobs:
 
       - name: Create kind cluster
         if: steps.list-changed.outputs.changed == 'true'
-        uses: helm/kind-action@v1.2.0
+        uses: helm/kind-action@v1.4.0
 
       # We need cert-manager already installed in the cluster because we assume the CRDs exist
       - name: Install cert-manager
@@ -86,20 +88,31 @@ jobs:
         if: steps.list-changed.outputs.changed == 'true'
         run: ct install --config charts/.ci/ct-config.yaml
 
-      # WARNING: This relies on the latest release being inat the top of the JSON from GitHub and a clean chart.yaml
+      # WARNING: This relies on the latest release being at the top of the JSON from GitHub and a clean chart.yaml
       - name: Check if Chart Publish is Needed
         id: publish-chart-step
         run: |
-          CHART_TEXT=$(curl -fs https://raw.githubusercontent.com/actions-runner-controller/actions-runner-controller/master/charts/actions-runner-controller/Chart.yaml)
+          CHART_TEXT=$(curl -fs https://raw.githubusercontent.com/${{ github.repository }}/master/charts/actions-runner-controller/Chart.yaml)
           NEW_CHART_VERSION=$(echo "$CHART_TEXT" | grep version: | cut -d ' ' -f 2)
-          RELEASE_LIST=$(curl -fs https://api.github.com/repos/actions-runner-controller/actions-runner-controller/releases  | jq .[].tag_name | grep actions-runner-controller | cut -d '"' -f 2 | cut -d '-' -f 4)
+          RELEASE_LIST=$(curl -fs https://api.github.com/repos/${{ github.repository }}/releases  | jq .[].tag_name | grep actions-runner-controller | cut -d '"' -f 2 | cut -d '-' -f 4)
           LATEST_RELEASED_CHART_VERSION=$(echo $RELEASE_LIST | cut -d ' ' -f 1)
-          echo "Chart version in master : $NEW_CHART_VERSION"
-          echo "Latest release chart version : $LATEST_RELEASED_CHART_VERSION"
+          echo "CHART_VERSION_IN_MASTER=$NEW_CHART_VERSION" >> $GITHUB_ENV
+          echo "LATEST_CHART_VERSION=$LATEST_RELEASED_CHART_VERSION" >> $GITHUB_ENV
           if [[ $NEW_CHART_VERSION != $LATEST_RELEASED_CHART_VERSION ]]; then
-            echo "::set-output name=publish::true"
+            echo "publish=true" >> $GITHUB_OUTPUT
+          else
+            echo "publish=false" >> $GITHUB_OUTPUT
           fi
 
+      - name: Job summary
+        run: |
+          echo "Chart linting has been completed." >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Status:**" >> $GITHUB_STEP_SUMMARY
+          echo "- chart version in master: ${{ env.CHART_VERSION_IN_MASTER }}" >> $GITHUB_STEP_SUMMARY
+          echo "- latest chart version: ${{ env.LATEST_CHART_VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "- publish new chart: ${{ steps.publish-chart-step.outputs.publish }}" >> $GITHUB_STEP_SUMMARY
+
   publish-chart:
     if: needs.lint-chart.outputs.publish-chart == 'true'
     needs: lint-chart
@@ -107,8 +120,11 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write  # for helm/chart-releaser-action to push chart release and create a release
+    env:
+      CHART_TARGET_ORG: actions-runner-controller
+      CHART_TARGET_REPO: actions-runner-controller.github.io
+      CHART_TARGET_BRANCH: main
     
-
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -120,8 +136,68 @@ jobs:
           git config user.name "$GITHUB_ACTOR"
           git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
 
-      - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.4.0
-        env:
-          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+      - name: Get Token
+        id: get_workflow_token
+        uses: peter-murray/workflow-application-token-action@8e1ba3bf1619726336414f1014e37f17fbadf1db
+        with:
+          application_id: ${{ secrets.ACTIONS_ACCESS_APP_ID }}
+          application_private_key: ${{ secrets.ACTIONS_ACCESS_PK }}
+          organization: ${{ env.CHART_TARGET_ORG }}
 
+      - name: Install chart-releaser
+        uses: helm/chart-releaser-action@v1.4.1
+        with:
+          install_only: true
+          install_dir: ${{ github.workspace }}/bin
+
+      - name: Package and upload release assets
+        run: |
+          cr package \
+            ${{ github.workspace }}/charts/actions-runner-controller/ \
+            --package-path .cr-release-packages
+
+          cr upload \
+            --owner "$(echo ${{ github.repository }} | cut -d '/' -f 1)" \
+            --git-repo "$(echo ${{ github.repository }} | cut -d '/' -f 2)" \
+            --package-path .cr-release-packages \
+            --token ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Generate updated index.yaml
+        run: |
+          cr index \
+            --owner "$(echo ${{ github.repository }} | cut -d '/' -f 1)" \
+            --git-repo "$(echo ${{ github.repository }} | cut -d '/' -f 2)" \
+            --index-path ${{ github.workspace }}/index.yaml \
+            --pages-branch 'gh-pages' \
+            --pages-index-path 'index.yaml'
+
+      # Chart Release was never intended to publish to a different repo
+      # this workaround is intended to move the index.yaml to the target repo
+      # where the github pages are hosted
+      - name: Checkout pages repository
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ env.CHART_TARGET_ORG }}/${{ env.CHART_TARGET_REPO }}
+          path: ${{ env.CHART_TARGET_REPO }}
+          ref: ${{ env.CHART_TARGET_BRANCH }}
+          token: ${{ steps.get_workflow_token.outputs.token }}
+
+      - name: Copy index.yaml
+        run: |
+          cp ${{ github.workspace }}/index.yaml ${{ env.CHART_TARGET_REPO }}/actions-runner-controller/index.yaml
+      
+      - name: Commit and push
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+          git add .
+          git commit -m "Update index.yaml"
+          git push
+        working-directory: ${{ github.workspace }}/${{ env.CHART_TARGET_REPO }}
+
+      - name: Job summary
+        run: |
+          echo "New helm chart has been published" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Status:**" >> $GITHUB_STEP_SUMMARY
+          echo "- New [index.yaml](https://github.com/${{ env.CHART_TARGET_ORG }}/${{ env.CHART_TARGET_REPO }}/tree/main/actions-runner-controller) pushed" >> $GITHUB_STEP_SUMMARY

.github/workflows/release-runners.yaml

@@ -0,0 +1,64 @@
+name: Runners
+
+# Revert to https://github.com/actions-runner-controller/releases#releases
+# for details on why we use this approach
+on:
+  # We must do a trigger on a push: instead of a types: closed so GitHub Secrets 
+  # are available to the workflow run
+  push:
+    branches:
+      - 'master'
+    paths:
+      - 'runner/**'
+      - '!runner/Makefile'
+      - '.github/workflows/runners.yaml'
+      - '!**.md'
+
+env:
+  # Safeguard to prevent pushing images to registeries after build  
+  PUSH_TO_REGISTRIES: true
+  TARGET_ORG: actions-runner-controller
+  TARGET_WORKFLOW: release-runners.yaml
+  RUNNER_VERSION: 2.300.2
+  DOCKER_VERSION: 20.10.21
+  RUNNER_CONTAINER_HOOKS_VERSION: 0.2.0
+
+jobs:
+  build-runners:
+    name: Trigger Build and Push of Runner Images
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get Token
+        id: get_workflow_token
+        uses: peter-murray/workflow-application-token-action@8e1ba3bf1619726336414f1014e37f17fbadf1db
+        with:
+          application_id: ${{ secrets.ACTIONS_ACCESS_APP_ID }}
+          application_private_key: ${{ secrets.ACTIONS_ACCESS_PK }}
+          organization: ${{ env.TARGET_ORG }}
+
+      - name: Trigger Build And Push Runner Images To Registries
+        run: |
+          # Authenticate
+          gh auth login --with-token <<< ${{ steps.get_workflow_token.outputs.token }}
+
+          # Trigger the workflow run
+          gh workflow run ${{ env.TARGET_WORKFLOW }} -R ${{ env.TARGET_ORG }}/releases \
+            -f runner_version=${{ env.RUNNER_VERSION }} \
+            -f docker_version=${{ env.DOCKER_VERSION }} \
+            -f runner_container_hooks_version=${{ env.RUNNER_CONTAINER_HOOKS_VERSION }} \
+            -f sha='${{ github.sha }}' \
+            -f push_to_registries=${{ env.PUSH_TO_REGISTRIES }}
+
+      - name: Job summary
+        run: |
+          echo "The [release-runners.yaml](https://github.com/actions-runner-controller/releases/blob/main/.github/workflows/release-runners.yaml) workflow has been triggered!" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Parameters:**" >> $GITHUB_STEP_SUMMARY
+          echo "- runner_version: ${{ env.RUNNER_VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "- docker_version: ${{ env.DOCKER_VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "- runner_container_hooks_version: ${{ env.RUNNER_CONTAINER_HOOKS_VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "- sha: ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY
+          echo "- push_to_registries: ${{ env.PUSH_TO_REGISTRIES }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Status:**" >> $GITHUB_STEP_SUMMARY
+          echo "[https://github.com/actions-runner-controller/releases/actions/workflows/release-runners.yaml](https://github.com/actions-runner-controller/releases/actions/workflows/release-runners.yaml)" >> $GITHUB_STEP_SUMMARY

.github/workflows/run-first-interaction.yaml

@@ -0,0 +1,29 @@
+name: first-interaction
+
+on:
+  issues:
+    types: [opened]
+  pull_request:
+    branches: [master]
+    types: [opened]
+
+jobs:
+  check_for_first_interaction:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/first-interaction@main
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          issue-message: |
+            Hello! Thank you for filing an issue.
+
+            The maintainers will triage your issue shortly.
+
+            In the meantime, please take a look at the [troubleshooting guide](https://github.com/actions/actions-runner-controller/blob/master/TROUBLESHOOTING.md) for bug reports.
+            
+            If this is a feature request, please review our [contribution guidelines](https://github.com/actions/actions-runner-controller/blob/master/CONTRIBUTING.md).
+          pr-message: |
+            Hello! Thank you for your contribution.
+
+            Please review our [contribution guidelines](https://github.com/actions/actions-runner-controller/blob/master/CONTRIBUTING.md) to understand the project's testing and code conventions.

.github/workflows/run-stale.yaml

@@ -14,7 +14,7 @@ jobs:
       issues: write         # for actions/stale to close stale issues
       pull-requests: write  # for actions/stale to close stale PRs
     steps:
-      - uses: actions/stale@v5
+      - uses: actions/stale@v6
         with:
           stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.'
           # turn off stale for both issues and PRs

.github/workflows/runners.yaml

@@ -1,72 +0,0 @@
-name: Runners
-
-on:
-  pull_request:
-    types:
-      - opened
-      - synchronize
-      - reopened
-      - closed
-    branches:
-      - 'master'
-    paths:
-      - 'runner/**'
-      - '!runner/Makefile'
-      - '.github/workflows/runners.yaml'
-      - '!**.md'
-
-env:
-  RUNNER_VERSION: 2.293.0
-  DOCKER_VERSION: 20.10.12
-  DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USER }}
-
-jobs:
-  build-runners:
-    name: Build ${{ matrix.name }}-${{ matrix.os-name }}-${{ matrix.os-version }}
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - name: actions-runner
-            os-name: ubuntu
-            os-version: 20.04
-          - name: actions-runner-dind
-            os-name: ubuntu
-            os-version: 20.04
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Setup Docker Environment
-        id: vars
-        uses: ./.github/actions/setup-docker-environment
-        with:
-          username: ${{ env.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
-          ghcr_username: ${{ github.actor }}
-          ghcr_password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and Push Versioned Tags
-        uses: docker/build-push-action@v3
-        with:
-          context: ./runner
-          file: ./runner/${{ matrix.name }}.dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
-          build-args: |
-            RUNNER_VERSION=${{ env.RUNNER_VERSION }}
-            DOCKER_VERSION=${{ env.DOCKER_VERSION }}
-          tags: |
-            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}
-            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}-${{ steps.vars.outputs.sha_short }}
-            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:latest
-            ghcr.io/${{ github.repository }}/${{ matrix.name }}:latest
-            ghcr.io/${{ github.repository }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}
-            ghcr.io/${{ github.repository }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}-${{ steps.vars.outputs.sha_short }}
-          cache-from: type=gha,scope=build-${{ matrix.name }}
-          cache-to: type=gha,mode=max,scope=build-${{ matrix.name }}

.github/workflows/validate-chart.yaml

@@ -26,7 +26,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v2.1
+        uses: azure/setup-helm@v3.4
         with:
           version: ${{ env.HELM_VERSION }}
 
@@ -52,7 +52,7 @@ jobs:
           python-version: '3.7'
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.2.1
+        uses: helm/chart-testing-action@v2.3.1
 
       - name: Run chart-testing (list-changed)
         id: list-changed
@@ -67,7 +67,7 @@ jobs:
           ct lint --config charts/.ci/ct-config.yaml
 
       - name: Create kind cluster
-        uses: helm/kind-action@v1.2.0
+        uses: helm/kind-action@v1.4.0
         if: steps.list-changed.outputs.changed == 'true'
 
       # We need cert-manager already installed in the cluster because we assume the CRDs exist

.github/workflows/validate-runners.yaml

@@ -6,13 +6,33 @@ on:
       - '**'
     paths:
       - 'runner/**'
-      - 'test/entrypoint/**'
+      - 'test/startup/**'
       - '!**.md'
 
 permissions:
   contents: read
 
 jobs:
+  shellcheck:
+    name: runner / shellcheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: shellcheck
+        uses: reviewdog/action-shellcheck@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          path: "./runner"
+          pattern: |
+            *.sh
+            *.bash
+            update-status
+          # Make this consistent with `make shellsheck`
+          shellcheck_flags: "--shell bash --source-path runner"
+          exclude: "./.git/*"
+          check_all_files_with_shebangs: "false"
+          # Set this to "true" once we addressed all the shellcheck findings
+          fail_on_error: "false"
   test-runner-entrypoint:
     name: Test entrypoint
     runs-on: ubuntu-latest
@@ -22,4 +42,4 @@ jobs:
 
     - name: Run tests
       run: |
-        make acceptance/runner/entrypoint
+        make acceptance/runner/startup

.golangci.yaml

@@ -0,0 +1,17 @@
+run:
+  timeout: 3m
+output:
+  format: github-actions
+linters-settings:
+  errcheck:
+    exclude-functions:
+      - (net/http.ResponseWriter).Write
+      - (*net/http.Server).Shutdown
+      - (*github.com/actions/actions-runner-controller/simulator.VisibleRunnerGroups).Add
+      - (*github.com/actions/actions-runner-controller/testing.Kind).Stop
+issues:
+  exclude-rules:
+    - path: controllers/suite_test.go
+      linters:
+        - staticcheck
+      text: "SA1019"

CODEOWNERS

@@ -1,2 +1,2 @@
 # actions-runner-controller maintainers
-* @mumoshu @toast-gear
+* @mumoshu @toast-gear @actions/actions-runtime

CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at opensource@github.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

CONTRIBUTING.md

@@ -1,85 +1,53 @@
-## Contributing
+# Contribution Guide
 
-### Testing Controller Built from a Pull Request
+- [Contribution Guide](#contribution-guide)
+  - [Welcome](#welcome)
+  - [Before contributing code](#before-contributing-code)
+  - [How to Contribute a Patch](#how-to-contribute-a-patch)
+    - [Developing the Controller](#developing-the-controller)
+    - [Developing the Runners](#developing-the-runners)
+      - [Tests](#tests)
+      - [Running Ginkgo Tests](#running-ginkgo-tests)
+    - [Running End to End Tests](#running-end-to-end-tests)
+      - [Rerunning a failed test](#rerunning-a-failed-test)
+      - [Testing in a non-kind cluster](#testing-in-a-non-kind-cluster)
+    - [Code conventions](#code-conventions)
+    - [Opening the Pull Request](#opening-the-pull-request)
+  - [Helm Version Changes](#helm-version-changes)
+  - [Testing Controller Built from a Pull Request](#testing-controller-built-from-a-pull-request)
 
-We always appreciate your help in testing open pull requests by deploying custom builds of actions-runner-controller onto your own environment, so that we are extra sure we didn't break anything.
+## Welcome
 
-It is especially true when the pull request is about GitHub Enterprise, both GHEC and GHES, as [maintainers don't have GitHub Enterprise environments for testing](/README.md#github-enterprise-support).
+This document is the single source of truth for how to contribute to the code base.
+Feel free to browse the [open issues](https://github.com/actions/actions-runner-controller/issues) or file a new one, all feedback is welcome!
+By reading this guide, we hope to give you all of the information you need to be able to pick up issues, contribute new features, and get your work
+reviewed and merged.
 
-The process would look like the below:
+## Before contributing code
 
-- Clone this repository locally
-- Checkout the branch. If you use the `gh` command, run `gh pr checkout $PR_NUMBER`
-- Run `NAME=$DOCKER_USER/actions-runner-controller VERSION=canary make docker-build docker-push` for a custom container image build
-- Update your actions-runner-controller's controller-manager deployment to use the new image, `$DOCKER_USER/actions-runner-controller:canary`
+We welcome code patches, but to make sure things are well coordinated you should  discuss any significant change before starting the work.
+The maintainers ask that you signal your intention to contribute to the project using the issue tracker. 
+If there is an existing issue that you want to work on, please let us know so we can get it assigned to you.
+If you noticed a bug or want to add a new feature, there are issue templates you can fill out.
 
-Please also note that you need to replace `$DOCKER_USER` with your own DockerHub account name.
+When filing a feature request, the maintainers will review the change and give you a decision on whether we are willing to accept the feature into the project.
+For significantly large and/or complex features, we may request that you write up an architectural decision record ([ADR](https://github.blog/2020-08-13-why-write-adrs/)) detailing the change.
+Please use the [template](/adrs/0000-TEMPLATE.md) as guidance.
 
-### How to Contribute a Patch
+<!-- 
+  TODO: Add a pre-requisite section describing what developers should
+  install in order get started on ARC.
+-->
 
-Depending on what you are patching depends on how you should go about it. Below are some guides on how to test patches locally as well as develop the controller and runners.
+## How to Contribute a Patch
 
-When submitting a PR for a change please provide evidence that your change works as we still need to work on improving the CI of the project. Some resources are provided for helping achieve this, see this guide for details.
+Depending on what you are patching depends on how you should go about it.
+Below are some guides on how to test patches locally as well as develop the controller and runners.
 
-#### Running an End to End Test
+When submitting a PR for a change please provide evidence that your change works as we still need to work on improving the CI of the project.
+Some resources are provided for helping achieve this, see this guide for details.
 
-> **Notes for Ubuntu 20.04+ users**
->
-> If you're using Ubuntu 20.04 or greater, you might have installed `docker` with `snap`.
->
-> If you want to stick with `snap`-provided `docker`, do not forget to set `TMPDIR` to
-> somewhere under `$HOME`.
-> Otherwise `kind load docker-image` fail while running `docker save`.
-> See https://kind.sigs.k8s.io/docs/user/known-issues/#docker-installed-with-snap for more information.
-
-To test your local changes against both PAT and App based authentication please run the `acceptance` make target with the authentication configuration details provided:
-
-```shell
-# This sets `VERSION` envvar to some appropriate value
-. hack/make-env.sh
-
-DOCKER_USER=*** \
-  GITHUB_TOKEN=*** \
-  APP_ID=*** \
-  PRIVATE_KEY_FILE_PATH=path/to/pem/file \
-  INSTALLATION_ID=*** \
-  make acceptance
-```
-
-**Rerunning a failed test**
-
-When one of tests run by `make acceptance` failed, you'd probably like to rerun only the failed one.
-
-It can be done by `make acceptance/run` and by setting the combination of `ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm|kubectl` and `ACCEPTANCE_TEST_SECRET_TYPE=token|app` values that failed (note, you just need to set the corresponding authentication configuration in this circumstance)
-
-In the example below, we rerun the test for the combination `ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm ACCEPTANCE_TEST_SECRET_TYPE=token` only:
-
-```shell
-DOCKER_USER=*** \
-  GITHUB_TOKEN=*** \
-  ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm
-  ACCEPTANCE_TEST_SECRET_TYPE=token \
-  make acceptance/run
-```
-
-**Testing in a non-kind cluster**
-
-If you prefer to test in a non-kind cluster, you can instead run:
-
-```shell
-KUBECONFIG=path/to/kubeconfig \
-  DOCKER_USER=*** \
-  GITHUB_TOKEN=*** \
-  APP_ID=*** \
-  PRIVATE_KEY_FILE_PATH=path/to/pem/file \
-  INSTALLATION_ID=*** \
-  ACCEPTANCE_TEST_SECRET_TYPE=token \
-  make docker-build acceptance/setup \
-       acceptance/deploy \
-       acceptance/tests
-```
-
-#### Developing the Controller
+### Developing the Controller
 
 Rerunning the whole acceptance test suite from scratch on every little change to the controller, the runner, and the chart would be counter-productive.
 
@@ -119,13 +87,14 @@ NAME=$DOCKER_USER/actions-runner make \
   (kubectl get po -ojsonpath={.items[*].metadata.name} | xargs -n1 kubectl delete po)
 ```
 
-#### Developing the Runners
+### Developing the Runners
 
-**Tests**
+#### Tests
 
-A set of example pipelines (./acceptance/pipelines) are provided in this repository which you can use to validate your runners are working as expected. When raising a PR please run the relevant suites to prove your change hasn't broken anything.
+A set of example pipelines (./acceptance/pipelines) are provided in this repository which you can use to validate your runners are working as expected.
+When raising a PR please run the relevant suites to prove your change hasn't broken anything.
 
-**Running Ginkgo Tests**
+#### Running Ginkgo Tests
 
 You can run the integration test suite that is written in Ginkgo with:
 
@@ -135,13 +104,14 @@ make test-with-deps
 
 This will firstly install a few binaries required to setup the integration test environment and then runs `go test` to start the Ginkgo test.
 
-If you don't want to use `make`, like when you're running tests from your IDE, install required binaries to `/usr/local/kubebuilder/bin`. That's the directory in which controller-runtime's `envtest` framework locates the binaries.
+If you don't want to use `make`, like when you're running tests from your IDE, install required binaries to `/usr/local/kubebuilder/bin`.
+That's the directory in which controller-runtime's `envtest` framework locates the binaries.
 
 ```shell
 sudo mkdir -p /usr/local/kubebuilder/bin
 make kube-apiserver etcd
 sudo mv test-assets/{etcd,kube-apiserver} /usr/local/kubebuilder/bin/
-go test -v -run TestAPIs github.com/actions-runner-controller/actions-runner-controller/controllers
+go test -v -run TestAPIs github.com/actions/actions-runner-controller/controllers/actions.summerwind.net
 ```
 
 To run Ginkgo tests selectively, set the pattern of target test names to `GINKGO_FOCUS`.
@@ -149,9 +119,101 @@ All the Ginkgo test that matches `GINKGO_FOCUS` will be run.
 
 ```shell
 GINKGO_FOCUS='[It] should create a new Runner resource from the specified template, add a another Runner on replicas increased, and removes all the replicas when set to 0' \
-  go test -v -run TestAPIs github.com/actions-runner-controller/actions-runner-controller/controllers
+  go test -v -run TestAPIs github.com/actions/actions-runner-controller/controllers/actions.summerwind.net
 ```
 
-#### Helm Version Bumps
+### Running End to End Tests
 
-In general we ask you not to bump the version in your PR, the maintainers in general manage the publishing of a new chart.
+> **Notes for Ubuntu 20.04+ users**
+>
+> If you're using Ubuntu 20.04 or greater, you might have installed `docker` with `snap`.
+>
+> If you want to stick with `snap`-provided `docker`, do not forget to set `TMPDIR` to somewhere under `$HOME`.
+> Otherwise `kind load docker-image` fail while running `docker save`.
+> See https://kind.sigs.k8s.io/docs/user/known-issues/#docker-installed-with-snap for more information.
+
+To test your local changes against both PAT and App based authentication please run the `acceptance` make target with the authentication configuration details provided:
+
+```shell
+# This sets `VERSION` envvar to some appropriate value
+. hack/make-env.sh
+
+DOCKER_USER=*** \
+  GITHUB_TOKEN=*** \
+  APP_ID=*** \
+  PRIVATE_KEY_FILE_PATH=path/to/pem/file \
+  INSTALLATION_ID=*** \
+  make acceptance
+```
+
+#### Rerunning a failed test
+
+When one of tests run by `make acceptance` failed, you'd probably like to rerun only the failed one.
+
+It can be done by `make acceptance/run` and by setting the combination of `ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm|kubectl` and `ACCEPTANCE_TEST_SECRET_TYPE=token|app` values that failed (note, you just need to set the corresponding authentication configuration in this circumstance)
+
+In the example below, we rerun the test for the combination `ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm ACCEPTANCE_TEST_SECRET_TYPE=token` only:
+
+```shell
+DOCKER_USER=*** \
+  GITHUB_TOKEN=*** \
+  ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm \
+  ACCEPTANCE_TEST_SECRET_TYPE=token \
+  make acceptance/run
+```
+
+#### Testing in a non-kind cluster
+
+If you prefer to test in a non-kind cluster, you can instead run:
+
+```shell
+KUBECONFIG=path/to/kubeconfig \
+  DOCKER_USER=*** \
+  GITHUB_TOKEN=*** \
+  APP_ID=*** \
+  PRIVATE_KEY_FILE_PATH=path/to/pem/file \
+  INSTALLATION_ID=*** \
+  ACCEPTANCE_TEST_SECRET_TYPE=token \
+  make docker-build acceptance/setup \
+       acceptance/deploy \
+       acceptance/tests
+```
+
+### Code conventions
+
+Before shipping your PR, please check the following items to make sure CI passes.
+
+- Run `go mod tidy` if you made changes to dependencies.
+- Format the code using `gofmt`
+- Run the `golangci-lint` tool locally.
+  -  We recommend you use `make lint` to run the tool using a Docker container matching the CI version.
+
+### Opening the Pull Request
+
+Send PR, add issue number to description
+
+## Helm Version Changes
+
+In general we ask you not to bump the version in your PR.
+The maintainers will manage releases and publishing new charts.
+
+## Testing Controller Built from a Pull Request
+
+We always appreciate your help in testing open pull requests by deploying custom builds of actions-runner-controller onto your own environment, so that we are extra sure we didn't break anything.
+
+It is especially true when the pull request is about GitHub Enterprise, both GHEC and GHES, as [maintainers don't have GitHub Enterprise environments for testing](docs/about-arc.md#github-enterprise-support).
+
+The process would look like the below:
+
+- Clone this repository locally
+- Checkout the branch. If you use the `gh` command, run `gh pr checkout $PR_NUMBER`
+- Run `NAME=$DOCKER_USER/actions-runner-controller VERSION=canary make docker-build docker-push` for a custom container image build
+- Update your actions-runner-controller's controller-manager deployment to use the new image, `$DOCKER_USER/actions-runner-controller:canary`
+
+Please also note that you need to replace `$DOCKER_USER` with your own DockerHub account name.
+
+## Release process
+
+Only the maintainers can release a new version of actions-runner-controller, publish a new version of the helm charts, and runner images.
+
+All release workflows have been moved to [actions-runner-controller/releases](https://github.com/actions-runner-controller/releases) since the packages are owned by the former organization.

Dockerfile

@@ -1,11 +1,10 @@
 # Build the manager binary
-FROM --platform=$BUILDPLATFORM golang:1.18.2 as builder
+FROM --platform=$BUILDPLATFORM golang:1.19.4 as builder
 
 WORKDIR /workspace
 
 # Make it runnable on a distroless image/without libc
 ENV CGO_ENABLED=0
-
 # Copy the Go Modules manifests
 COPY go.mod go.sum ./
 
@@ -25,20 +24,21 @@ RUN go mod download
 # With the above commmand,
 # TARGETOS can be "linux", TARGETARCH can be "amd64", "arm64", and "arm", TARGETVARIANT can be "v7".
 
-ARG TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT
+ARG TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT VERSION=dev
 
 # We intentionally avoid `--mount=type=cache,mode=0777,target=/go/pkg/mod` in the `go mod download` and the `go build` runs
 # to avoid https://github.com/moby/buildkit/issues/2334
 # We can use docker layer cache so the build is fast enogh anyway
 # We also use per-platform GOCACHE for the same reason.
-env GOCACHE /build/${TARGETPLATFORM}/root/.cache/go-build
+ENV GOCACHE /build/${TARGETPLATFORM}/root/.cache/go-build
 
 # Build
 RUN --mount=target=. \
   --mount=type=cache,mode=0777,target=${GOCACHE} \
   export GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOARM=${TARGETVARIANT#v} && \
-  go build -o /out/manager main.go && \
-  go build -o /out/github-webhook-server ./cmd/githubwebhookserver
+  go build -trimpath -ldflags="-s -w -X 'github.com/actions/actions-runner-controller/build.Version=${VERSION}'" -o /out/manager main.go && \
+  go build -trimpath -ldflags="-s -w" -o /out/github-webhook-server ./cmd/githubwebhookserver && \
+  go build -trimpath -ldflags="-s -w" -o /out/actions-metrics-server ./cmd/actionsmetricsserver
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
@@ -48,7 +48,8 @@ WORKDIR /
 
 COPY --from=builder /out/manager .
 COPY --from=builder /out/github-webhook-server .
+COPY --from=builder /out/actions-metrics-server .
 
-USER nonroot:nonroot
+USER 65532:65532
 
 ENTRYPOINT ["/manager"]

Makefile

@@ -4,8 +4,8 @@ else
 	NAME ?= summerwind/actions-runner-controller
 endif
 DOCKER_USER ?= $(shell echo ${NAME} | cut -d / -f1)
-VERSION ?= latest
-RUNNER_VERSION ?= 2.293.0
+VERSION ?= dev
+RUNNER_VERSION ?= 2.300.2
 TARGETPLATFORM ?= $(shell arch)
 RUNNER_NAME ?= ${DOCKER_USER}/actions-runner
 RUNNER_TAG  ?= ${VERSION}
@@ -19,6 +19,7 @@ KUBECONTEXT ?= kind-acceptance
 CLUSTER ?= acceptance
 CERT_MANAGER_VERSION ?= v1.1.1
 KUBE_RBAC_PROXY_VERSION ?= v0.11.0
+SHELLCHECK_VERSION ?= 0.8.0
 
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:generateEmbeddedObjectMeta=true"
@@ -31,6 +32,20 @@ GOBIN=$(shell go env GOBIN)
 endif
 
 TEST_ASSETS=$(PWD)/test-assets
+TOOLS_PATH=$(PWD)/.tools
+
+OS_NAME := $(shell uname -s | tr A-Z a-z)
+
+# The etcd packages that coreos maintain use different extensions for each *nix OS on their github release page.
+# ETCD_EXTENSION: the storage format file extension listed on the release page.
+# EXTRACT_COMMAND: the  appropriate CLI command for extracting this file format.
+ifeq ($(OS_NAME), darwin)
+ETCD_EXTENSION:=zip
+EXTRACT_COMMAND:=unzip
+else
+ETCD_EXTENSION:=tar.gz
+EXTRACT_COMMAND:=tar -xzf
+endif
 
 # default list of platforms for which multiarch image is built
 ifeq (${PLATFORMS}, )
@@ -51,12 +66,15 @@ endif
 
 all: manager
 
+lint:
+	docker run --rm -v $(PWD):/app -w /app golangci/golangci-lint:v1.49.0 golangci-lint run
+
 GO_TEST_ARGS ?= -short
 
 # Run tests
-test: generate fmt vet manifests
+test: generate fmt vet manifests shellcheck
 	go test $(GO_TEST_ARGS) ./... -coverprofile cover.out
-	go test -fuzz=Fuzz -fuzztime=10s -run=Fuzz* ./controllers
+	go test -fuzz=Fuzz -fuzztime=10s -run=Fuzz* ./controllers/actions.summerwind.net
 
 test-with-deps: kube-apiserver etcd kubectl
 	# See https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest#pkg-constants
@@ -92,7 +110,7 @@ manifests: manifests-gen-crds chart-crds
 manifests-gen-crds: controller-gen yq
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 	for YAMLFILE in config/crd/bases/actions*.yaml; do \
-		$(YQ) write --inplace "$$YAMLFILE" spec.preserveUnknownFields false; \
+		$(YQ) '.spec.preserveUnknownFields = false' --inplace "$$YAMLFILE" ; \
 	done
 
 chart-crds:
@@ -110,6 +128,10 @@ vet:
 generate: controller-gen
 	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."
 
+# Run shellcheck on runner scripts
+shellcheck: shellcheck-install
+	$(TOOLS_PATH)/shellcheck --shell bash --source-path runner runner/*.sh
+
 docker-buildx:
 	export DOCKER_CLI_EXPERIMENTAL=enabled ;\
 	export DOCKER_BUILDKIT=1
@@ -119,6 +141,7 @@ docker-buildx:
 	docker buildx build --platform ${PLATFORMS} \
 		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
 		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		--build-arg VERSION=${VERSION} \
 		-t "${NAME}:${VERSION}" \
 		-f Dockerfile \
 		. ${PUSH_ARG}
@@ -193,8 +216,8 @@ acceptance/deploy:
 acceptance/tests:
 	acceptance/checks.sh
 
-acceptance/runner/entrypoint:
-	cd test/entrypoint/ && bash test.sh
+acceptance/runner/startup:
+	cd test/startup/ && bash test.sh
 
 # We use -count=1 instead of `go clean -testcache`
 # See https://terratest.gruntwork.io/docs/testing-best-practices/avoid-test-caching/
@@ -242,13 +265,30 @@ ifeq (, $(wildcard $(GOBIN)/yq))
 	YQ_TMP_DIR=$$(mktemp -d) ;\
 	cd $$YQ_TMP_DIR ;\
 	go mod init tmp ;\
-	go install github.com/mikefarah/yq/v3@3.4.0 ;\
+	go install github.com/mikefarah/yq/v4@v4.25.3 ;\
 	rm -rf $$YQ_TMP_DIR ;\
 	}
 endif
 YQ=$(GOBIN)/yq
 
-OS_NAME := $(shell uname -s | tr A-Z a-z)
+# find or download shellcheck
+# download shellcheck if necessary
+shellcheck-install:
+ifeq (, $(wildcard $(TOOLS_PATH)/shellcheck))
+	echo "Downloading shellcheck"
+	@{ \
+	set -e ;\
+	SHELLCHECK_TMP_DIR=$$(mktemp -d) ;\
+	cd $$SHELLCHECK_TMP_DIR ;\
+	curl -LO https://github.com/koalaman/shellcheck/releases/download/v$(SHELLCHECK_VERSION)/shellcheck-v$(SHELLCHECK_VERSION).$(OS_NAME).x86_64.tar.xz ;\
+	tar Jxvf shellcheck-v$(SHELLCHECK_VERSION).$(OS_NAME).x86_64.tar.xz ;\
+	cd $(CURDIR) ;\
+	mkdir -p $(TOOLS_PATH) ;\
+	mv $$SHELLCHECK_TMP_DIR/shellcheck-v$(SHELLCHECK_VERSION)/shellcheck $(TOOLS_PATH)/ ;\
+	rm -rf $$SHELLCHECK_TMP_DIR ;\
+	}
+endif
+SHELLCHECK=$(TOOLS_PATH)/shellcheck
 
 # find or download etcd
 etcd:
@@ -258,12 +298,10 @@ ifeq (, $(wildcard $(TEST_ASSETS)/etcd))
 	set -xe ;\
 	INSTALL_TMP_DIR=$$(mktemp -d) ;\
 	cd $$INSTALL_TMP_DIR ;\
-	wget https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.2/kubebuilder_2.3.2_$(OS_NAME)_amd64.tar.gz ;\
+	wget https://github.com/coreos/etcd/releases/download/v3.4.22/etcd-v3.4.22-$(OS_NAME)-amd64.$(ETCD_EXTENSION);\
 	mkdir -p $(TEST_ASSETS) ;\
-	tar zxvf kubebuilder_2.3.2_$(OS_NAME)_amd64.tar.gz ;\
-	mv kubebuilder_2.3.2_$(OS_NAME)_amd64/bin/etcd $(TEST_ASSETS)/etcd ;\
-	mv kubebuilder_2.3.2_$(OS_NAME)_amd64/bin/kube-apiserver $(TEST_ASSETS)/kube-apiserver ;\
-	mv kubebuilder_2.3.2_$(OS_NAME)_amd64/bin/kubectl $(TEST_ASSETS)/kubectl ;\
+	$(EXTRACT_COMMAND) etcd-v3.4.22-$(OS_NAME)-amd64.$(ETCD_EXTENSION) ;\
+	mv etcd-v3.4.22-$(OS_NAME)-amd64/etcd $(TEST_ASSETS)/etcd ;\
 	rm -rf $$INSTALL_TMP_DIR ;\
 	}
 ETCD_BIN=$(TEST_ASSETS)/etcd
@@ -285,9 +323,7 @@ ifeq (, $(wildcard $(TEST_ASSETS)/kube-apiserver))
 	wget https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.2/kubebuilder_2.3.2_$(OS_NAME)_amd64.tar.gz ;\
 	mkdir -p $(TEST_ASSETS) ;\
 	tar zxvf kubebuilder_2.3.2_$(OS_NAME)_amd64.tar.gz ;\
-	mv kubebuilder_2.3.2_$(OS_NAME)_amd64/bin/etcd $(TEST_ASSETS)/etcd ;\
 	mv kubebuilder_2.3.2_$(OS_NAME)_amd64/bin/kube-apiserver $(TEST_ASSETS)/kube-apiserver ;\
-	mv kubebuilder_2.3.2_$(OS_NAME)_amd64/bin/kubectl $(TEST_ASSETS)/kubectl ;\
 	rm -rf $$INSTALL_TMP_DIR ;\
 	}
 KUBE_APISERVER_BIN=$(TEST_ASSETS)/kube-apiserver
@@ -309,8 +345,6 @@ ifeq (, $(wildcard $(TEST_ASSETS)/kubectl))
 	wget https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.2/kubebuilder_2.3.2_$(OS_NAME)_amd64.tar.gz ;\
 	mkdir -p $(TEST_ASSETS) ;\
 	tar zxvf kubebuilder_2.3.2_$(OS_NAME)_amd64.tar.gz ;\
-	mv kubebuilder_2.3.2_$(OS_NAME)_amd64/bin/etcd $(TEST_ASSETS)/etcd ;\
-	mv kubebuilder_2.3.2_$(OS_NAME)_amd64/bin/kube-apiserver $(TEST_ASSETS)/kube-apiserver ;\
 	mv kubebuilder_2.3.2_$(OS_NAME)_amd64/bin/kubectl $(TEST_ASSETS)/kubectl ;\
 	rm -rf $$INSTALL_TMP_DIR ;\
 	}

PROJECT

@@ -1,5 +1,5 @@
 domain: summerwind.dev
-repo: github.com/actions-runner-controller/actions-runner-controller
+repo: github.com/actions/actions-runner-controller
 resources:
 - group: actions
   kind: Runner

SECURITY.md

@@ -1,22 +1,31 @@
-# Security Policy
+Thanks for helping make GitHub safe for everyone.
 
-##  Sponsoring the project
+## Security
 
-This project is maintained by a small team of two and therefore lacks the resource to provide security fixes in a timely manner.
+GitHub takes the security of our software products and services seriously, including all of the open source code repositories managed through our GitHub organizations, such as [GitHub](https://github.com/GitHub).
 
-If you have important business(es) that relies on this project, please consider sponsoring the project so that the maintainer(s) can commit to providing such service.
+Even though [open source repositories are outside of the scope of our bug bounty program](https://bounty.github.com/index.html#scope) and therefore not eligible for bounty rewards, we will ensure that your finding gets passed along to the appropriate maintainers for remediation. 
 
-Please refer to https://github.com/sponsors/actions-runner-controller for available tiers.
+## Reporting Security Issues
 
-## Supported Versions
+If you believe you have found a security vulnerability in any GitHub-owned repository, please report it to us through coordinated disclosure.
 
-| Version | Supported          |
-| ------- | ------------------ |
-| 0.23.0  | :white_check_mark: |
-| < 0.23.0| :x:                |
+**Please do not report security vulnerabilities through public GitHub issues, discussions, or pull requests.**
 
-## Reporting a Vulnerability
+Instead, please send an email to opensource-security[@]github.com.
 
-To report a security issue, please email ykuoka+arcsecurity(at)gmail.com with a description of the issue, the steps you took to create the issue, affected versions, and, if known, mitigations for the issue.
+Please include as much of the information listed below as you can to help us better understand and resolve the issue:
 
-A maintainer will try to respond within 5 working days. If the issue is confirmed as a vulnerability, a Security Advisory will be opened. This project tries to follow a 90 day disclosure timeline.
+  * The type of issue (e.g., buffer overflow, SQL injection, or cross-site scripting)
+  * Full paths of source file(s) related to the manifestation of the issue
+  * The location of the affected source code (tag/branch/commit or direct URL)
+  * Any special configuration required to reproduce the issue
+  * Step-by-step instructions to reproduce the issue
+  * Proof-of-concept or exploit code (if possible)
+  * Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+## Policy
+
+See [GitHub's Safe Harbor Policy](https://docs.github.com/en/github/site-policy/github-bug-bounty-program-legal-safe-harbor#1-safe-harbor-terms)

TROUBLESHOOTING.md

@@ -2,14 +2,16 @@
 
 * [Tools](#tools)
 * [Installation](#installation)
+  * [InternalError when calling webhook: context deadline exceeded](#internalerror-when-calling-webhook-context-deadline-exceeded)
   * [Invalid header field value](#invalid-header-field-value)
-  * [Deployment fails on GKE due to webhooks](#deployment-fails-on-gke-due-to-webhooks)
+  * [Helm chart install failure: certificate signed by unknown authority](#helm-chart-install-failure-certificate-signed-by-unknown-authority)
 * [Operations](#operations)
   * [Stuck runner kind or backing pod](#stuck-runner-kind-or-backing-pod)
   * [Delay in jobs being allocated to runners](#delay-in-jobs-being-allocated-to-runners)
   * [Runner coming up before network available](#runner-coming-up-before-network-available)
   * [Outgoing network action hangs indefinitely](#outgoing-network-action-hangs-indefinitely)
   * [Unable to scale to zero with TotalNumberOfQueuedAndInProgressWorkflowRuns](#unable-to-scale-to-zero-with-totalnumberofqueuedandinprogressworkflowruns)
+  * [Slow / failure to boot dind sidecar (default runner)](#slow--failure-to-boot-dind-sidecar-default-runner)
 
 ## Tools
 
@@ -22,39 +24,37 @@ A list of tools which are helpful for troubleshooting
 
 Troubeshooting runbooks that relate to ARC installation problems
 
-### Invalid header field value
+### InternalError when calling webhook: context deadline exceeded
 
 **Problem**
 
-```json
-2020-11-12T22:17:30.693Z	ERROR	controller-runtime.controller	Reconciler error	
-{
-  "controller": "runner",
-  "request": "actions-runner-system/runner-deployment-dk7q8-dk5c9",
-  "error": "failed to create registration token: Post \"https://api.github.com/orgs/$YOUR_ORG_HERE/actions/runners/registration-token\": net/http: invalid header field value \"Bearer $YOUR_TOKEN_HERE\\n\" for key Authorization"
-}
+This issue can come up for various reasons like leftovers from previous installations or not being able to access the K8s service's clusterIP associated with the admission webhook server (of ARC).
+
+```
+Internal error occurred: failed calling webhook "mutate.runnerdeployment.actions.summerwind.dev":
+Post "https://actions-runner-controller-webhook.actions-runner-system.svc:443/mutate-actions-summerwind-dev-v1alpha1-runnerdeployment?timeout=10s": context deadline exceeded
 ```
 
 **Solution**
 
-Your base64'ed PAT token has a new line at the end, it needs to be created without a `\n` added, either:
-* `echo -n $TOKEN | base64`
-* Create the secret as described in the docs using the shell and documented flags
+First we will try the common solution of checking webhook leftovers from previous installations:
+
+1.  ```bash
+    kubectl get validatingwebhookconfiguration -A
+    kubectl get mutatingwebhookconfiguration -A
+    ```
+2.  If you see any webhooks related to actions-runner-controller, delete them:
+    ```bash
+    kubectl delete mutatingwebhookconfiguration actions-runner-controller-mutating-webhook-configuration
+    kubectl delete validatingwebhookconfiguration actions-runner-controller-validating-webhook-configuration
+    ```
+
+If that didn't work then probably your K8s control-plane is somehow unable to access the K8s service's clusterIP associated with the admission webhook server:
+1. You're running apiserver as a binary and you didn't make service cluster IPs available to the host network. 
+2. You're running the apiserver in the pod but your pod network (i.e. CNI plugin installation and config) is not good so your pods(like kube-apiserver) in the K8s control-plane nodes can't access ARC's admission webhook server pod(s) in probably data-plane nodes.
 
 
-### Deployment fails on GKE due to webhooks
-
-**Problem**
-
-Due to GKEs firewall settings you may run into the following errors when trying to deploy runners on a private GKE cluster:
-
-```
-Internal error occurred: failed calling webhook "mutate.runner.actions.summerwind.dev": 
-Post https://webhook-service.actions-runner-system.svc:443/mutate-actions-summerwind-dev-v1alpha1-runner?timeout=10s: 
-context deadline exceeded
-```
-
-**Solution**<br />
+Another reason could be due to GKEs firewall settings you may run into the following errors when trying to deploy runners on a private GKE cluster: 
 
 To fix this, you may either:
 
@@ -63,9 +63,9 @@ To fix this, you may either:
 
    ```sh
    # With helm, you'd set `webhookPort` to the port number of your choice
-   # See https://github.com/actions-runner-controller/actions-runner-controller/pull/1410/files for more information
+   # See https://github.com/actions/actions-runner-controller/pull/1410/files for more information
    helm upgrade --install --namespace actions-runner-system --create-namespace \
-                --wait actions-runner-controller actions-runner-controller/actions-runner-controller \
+                --wait actions-runner-controller actions/actions-runner-controller \
                 --set webhookPort=10250
    ```
 
@@ -88,6 +88,57 @@ To fix this, you may either:
    gcloud compute firewall-rules create k8s-cert-manager --source-ranges $SOURCE --target-tags $WORKER_NODES_TAG  --allow TCP:9443 --network $NETWORK
    ```
 
+### Invalid header field value
+
+**Problem**
+
+```json
+2020-11-12T22:17:30.693Z	ERROR	controller-runtime.controller	Reconciler error	
+{
+  "controller": "runner",
+  "request": "actions-runner-system/runner-deployment-dk7q8-dk5c9",
+  "error": "failed to create registration token: Post \"https://api.github.com/orgs/$YOUR_ORG_HERE/actions/runners/registration-token\": net/http: invalid header field value \"Bearer $YOUR_TOKEN_HERE\\n\" for key Authorization"
+}
+```
+
+**Solution**
+
+Your base64'ed PAT token has a new line at the end, it needs to be created without a `\n` added, either:
+* `echo -n $TOKEN | base64`
+* Create the secret as described in the docs using the shell and documented flags
+
+### Helm chart install failure: certificate signed by unknown authority
+
+**Problem**
+
+```
+Error: UPGRADE FAILED: failed to create resource: Internal error occurred: failed calling webhook "webhook.cert-manager.io": failed to call webhook: Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": x509: certificate signed by unknown authority
+```
+
+Apparently, it's failing while `helm` is creating one of resources defined in the ARC chart and the cause was that cert-manager's webhook is not working correctly, due to the missing or the invalid CA certficate.
+
+You'd try to tail logs from the `cert-manager-cainjector` and see it's failing with an error like:
+
+```
+$ kubectl -n cert-manager logs cert-manager-cainjector-7cdbb9c945-g6bt4
+I0703 03:31:55.159339       1 start.go:91] "starting" version="v1.1.1" revision="3ac7418070e22c87fae4b22603a6b952f797ae96"
+I0703 03:31:55.615061       1 leaderelection.go:243] attempting to acquire leader lease  kube-system/cert-manager-cainjector-leader-election...
+I0703 03:32:10.738039       1 leaderelection.go:253] successfully acquired lease kube-system/cert-manager-cainjector-leader-election
+I0703 03:32:10.739941       1 recorder.go:52] cert-manager/controller-runtime/manager/events "msg"="Normal"  "message"="cert-manager-cainjector-7cdbb9c945-g6bt4_88e4bc70-eded-4343-a6fb-0ddd6434eb55 became leader" "object"={"kind":"ConfigMap","namespace":"kube-system","name":"cert-manager-cainjector-leader-election","uid":"942a021e-364c-461a-978c-f54a95723cdc","apiVersion":"v1","resourceVersion":"1576"} "reason"="LeaderElection"
+E0703 03:32:11.192128       1 start.go:119] cert-manager/ca-injector "msg"="manager goroutine exited" "error"=null
+I0703 03:32:12.339197       1 request.go:645] Throttling request took 1.047437675s, request: GET:https://10.96.0.1:443/apis/storage.k8s.io/v1beta1?timeout=32s
+E0703 03:32:13.143790       1 start.go:151] cert-manager/ca-injector "msg"="Error registering certificate based controllers. Retrying after 5 seconds." "error"="no matches for kind \"MutatingWebhookConfiguration\" in version \"admissionregistration.k8s.io/v1beta1\""
+Error: error registering secret controller: no matches for kind "MutatingWebhookConfiguration" in version "admissionregistration.k8s.io/v1beta1"
+```
+
+**Solution**
+
+Your cluster is based on a new enough Kubernetes of version 1.22 or greater which does not support the legacy `admissionregistration.k8s.io/v1beta1` API anymore, and your `cert-manager` is not up-to-date hence it's still trying to use the leagcy Kubernetes API.
+
+In many cases, it's not an option to downgrade Kubernetes. So, just upgrade `cert-manager` to a more recent version that does have have the support for the specific Kubernetes version you're using.
+
+See https://cert-manager.io/docs/installation/supported-releases/ for the list of available cert-manager versions.
+
 ## Operations
 
 Troubeshooting runbooks that relate to ARC operational problems
@@ -117,7 +168,7 @@ are in a namespace not shared with anything else_
 
 **Problem**
 
-ARC isn't involved in jobs actually getting allocated to a runner. ARC is responsible for orchestrating runners and the runner lifecycle. Why some people see large delays in job allocation is not clear however it has been https://github.com/actions-runner-controller/actions-runner-controller/issues/1387#issuecomment-1122593984 that this is caused from the self-update process somehow.
+ARC isn't involved in jobs actually getting allocated to a runner. ARC is responsible for orchestrating runners and the runner lifecycle. Why some people see large delays in job allocation is not clear however it has been confirmed https://github.com/actions/actions-runner-controller/issues/1387#issuecomment-1122593984 that this is caused from the self-update process somehow.
 
 **Solution**
 
@@ -163,7 +214,7 @@ More broadly, there are many other circumstances where the runner pod coming up
 
 > Added originally to help users with older istio instances.
 > Newer Istio instances can use Istio's `holdApplicationUntilProxyStarts` attribute ([istio/istio#11130](https://github.com/istio/istio/issues/11130)) to avoid having to delay starting up the runner.
-> Please read the discussion in [#592](https://github.com/actions-runner-controller/actions-runner-controller/pull/592) for more information.
+> Please read the discussion in [#592](https://github.com/actions/actions-runner-controller/pull/592) for more information.
 
 You can add a delay to the runner's entrypoint script by setting the `STARTUP_DELAY_IN_SECONDS` environment variable for the runner pod. This will cause the script to sleep X seconds, this works with any runner kind.
 
@@ -206,8 +257,28 @@ spec:
       env: []
 ```
 
-There may be more places you need to tweak for MTU.
-Please consult issues like #651 for more information.
+If the issue still persists, you can set the `ARC_DOCKER_MTU_PROPAGATION` to propagate the host MTU to networks created
+by the GitHub Runner. For instance:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: github-runner
+  namespace: github-system
+spec:
+  replicas: 6
+  template:
+    spec:
+      dockerMTU: 1400
+      repository: $username/$repo
+      env:
+        - name: ARC_DOCKER_MTU_PROPAGATION
+          value: "true"
+```
+
+You can read the discussion regarding this issue in
+(#1406)[https://github.com/actions/actions-runner-controller/issues/1046].
 
 ## Unable to scale to zero with TotalNumberOfQueuedAndInProgressWorkflowRuns
 
@@ -217,6 +288,16 @@ HRA doesn't scale the RunnerDeployment to zero, even though you did configure HR
 
 **Solution**
 
-You very likely have some dangling workflow jobs stuck in `queued` or `in_progress` as seen in [#1057](https://github.com/actions-runner-controller/actions-runner-controller/issues/1057#issuecomment-1133439061).
+You very likely have some dangling workflow jobs stuck in `queued` or `in_progress` as seen in [#1057](https://github.com/actions/actions-runner-controller/issues/1057#issuecomment-1133439061).
 
 Manually call [the "list workflow runs" API](https://docs.github.com/en/rest/actions/workflow-runs#list-workflow-runs-for-a-repository), and [remove the dangling workflow job(s)](https://docs.github.com/en/rest/actions/workflow-runs#delete-a-workflow-run).
+
+## Slow / failure to boot dind sidecar (default runner)
+
+**Problem**
+
+If you noticed that it takes several minutes for sidecar dind container to be created or it exits with with error just after being created it might indicate that you are experiencing disk performance issue. You might see message `failed to reserve container name` when scaling up multiple runners at once. When you ssh on kubernetes node that problematic pods were scheduled on you can use tools like `atop`, `htop` or `iotop` to check IO usage and cpu time percentage used on iowait. If you see that disk usage is high (80-100%) and iowaits are taking a significant chunk of you cpu time (normally it should not be higher than 10%) it means that performance is being bottlenecked by slow disk.
+
+**Solution**
+
+The solution is to switch to using faster storage, if you are experiencing this issue you are probably using hdd, switch to ssh fixed the problem in my case. Most cloud providers have a list of storage options to use just pick something faster that your current disk, for on prem clusters you will need to invest in some ssds.

acceptance/argotunnel.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+
+# See https://developers.cloudflare.com/cloudflare-one/tutorials/many-cfd-one-tunnel/
+
+kubectl create ns tunnel || :
+
+kubectl -n tunnel delete secret tunnel-credentials || :
+
+kubectl -n tunnel create secret generic tunnel-credentials \
+  --from-file=credentials.json=$HOME/.cloudflared/${TUNNEL_ID}.json || :
+
+cat <<MANIFEST | kubectl -n tunnel ${OP} -f -
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cloudflared
+spec:
+  selector:
+    matchLabels:
+      app: cloudflared
+  replicas: 2 # You could also consider elastic scaling for this deployment
+  template:
+    metadata:
+      labels:
+        app: cloudflared
+    spec:
+      containers:
+      - name: cloudflared
+        image: cloudflare/cloudflared:latest
+        args:
+        - tunnel
+        # Points cloudflared to the config file, which configures what
+        # cloudflared will actually do. This file is created by a ConfigMap
+        # below.
+        - --config
+        - /etc/cloudflared/config/config.yaml
+        - run
+        livenessProbe:
+          httpGet:
+            # Cloudflared has a /ready endpoint which returns 200 if and only if
+            # it has an active connection to the edge.
+            path: /ready
+            port: 2000
+          failureThreshold: 1
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        volumeMounts:
+        - name: config
+          mountPath: /etc/cloudflared/config
+          readOnly: true
+        # Each tunnel has an associated "credentials file" which authorizes machines
+        # to run the tunnel. cloudflared will read this file from its local filesystem,
+        # and it'll be stored in a k8s secret.
+        - name: creds
+          mountPath: /etc/cloudflared/creds
+          readOnly: true
+      volumes:
+      - name: creds
+        secret:
+          secretName: tunnel-credentials
+      # Create a config.yaml file from the ConfigMap below.
+      - name: config
+        configMap:
+          name: cloudflared
+          items:
+          - key: config.yaml
+            path: config.yaml
+---
+# This ConfigMap is just a way to define the cloudflared config.yaml file in k8s.
+# It's useful to define it in k8s, rather than as a stand-alone .yaml file, because
+# this lets you use various k8s templating solutions (e.g. Helm charts) to
+# parameterize your config, instead of just using string literals.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cloudflared
+data:
+  config.yaml: |
+    # Name of the tunnel you want to run
+    tunnel: ${TUNNEL_NAME}
+    credentials-file: /etc/cloudflared/creds/credentials.json
+    # Serves the metrics server under /metrics and the readiness server under /ready
+    metrics: 0.0.0.0:2000
+    # Autoupdates applied in a k8s pod will be lost when the pod is removed or restarted, so
+    # autoupdate doesn't make sense in Kubernetes. However, outside of Kubernetes, we strongly
+    # recommend using autoupdate.
+    no-autoupdate: true
+    ingress:
+    # The first rule proxies traffic to the httpbin sample Service defined in app.yaml
+    - hostname: ${TUNNEL_HOSTNAME}
+      service: http://actions-runner-controller-actions-metrics-server.actions-runner-system:80
+      path: /metrics$
+    - hostname: ${TUNNEL_HOSTNAME}
+      service: http://actions-runner-controller-github-webhook-server.actions-runner-system:80
+    # This rule matches any traffic which didn't match a previous rule, and responds with HTTP 404.
+    - service: http_status:404
+MANIFEST
+
+kubectl -n tunnel delete po -l app=cloudflared || :

acceptance/deploy.sh

@@ -35,14 +35,45 @@ else
   echo 'Skipped deploying secret "github-webhook-server". Set WEBHOOK_GITHUB_TOKEN to deploy.' 1>&2
 fi
 
+if [ -n "${WEBHOOK_GITHUB_TOKEN}" ]; then
+  kubectl -n actions-runner-system delete secret \
+      actions-metrics-server || :
+  kubectl -n actions-runner-system create secret generic \
+      actions-metrics-server \
+      --from-literal=github_token=${WEBHOOK_GITHUB_TOKEN:?WEBHOOK_GITHUB_TOKEN must not be empty}
+else
+  echo 'Skipped deploying secret "actions-metrics-server". Set WEBHOOK_GITHUB_TOKEN to deploy.' 1>&2
+fi
+
 tool=${ACCEPTANCE_TEST_DEPLOYMENT_TOOL}
 
 TEST_ID=${TEST_ID:-default}
 
 if [ "${tool}" == "helm" ]; then
   set -v
+
+  CHART=${CHART:-charts/actions-runner-controller}
+
+  flags=()
+  if [ "${IMAGE_PULL_SECRET}" != "" ]; then
+    flags+=( --set imagePullSecrets[0].name=${IMAGE_PULL_SECRET})
+    flags+=( --set image.actionsRunnerImagePullSecrets[0].name=${IMAGE_PULL_SECRET})
+    flags+=( --set githubWebhookServer.imagePullSecrets[0].name=${IMAGE_PULL_SECRET})
+    flags+=( --set actionsMetricsServer.imagePullSecrets[0].name=${IMAGE_PULL_SECRET})
+  fi
+  if [ "${CHART_VERSION}" != "" ]; then
+    flags+=( --version ${CHART_VERSION})
+  fi
+  if [ "${LOG_FORMAT}" != "" ]; then
+    flags+=( --set logFormat=${LOG_FORMAT})
+    flags+=( --set githubWebhookServer.logFormat=${LOG_FORMAT})
+    flags+=( --set actionsMetricsServer.logFormat=${LOG_FORMAT})
+  fi
+
+  set -vx
+
   helm upgrade --install actions-runner-controller \
-    charts/actions-runner-controller \
+    ${CHART} \
     -n actions-runner-system \
     --create-namespace \
     --set syncPeriod=${SYNC_PERIOD} \
@@ -51,6 +82,8 @@ if [ "${tool}" == "helm" ]; then
     --set image.tag=${VERSION} \
     --set podAnnotations.test-id=${TEST_ID} \
     --set githubWebhookServer.podAnnotations.test-id=${TEST_ID} \
+    --set actionsMetricsServer.podAnnotations.test-id=${TEST_ID} \
+    ${flags[@]} --set image.imagePullPolicy=${IMAGE_PULL_POLICY} \
     -f ${VALUES_FILE}
   set +v
   # To prevent `CustomResourceDefinition.apiextensions.k8s.io "runners.actions.summerwind.dev" is invalid: metadata.annotations: Too long: must have at most 262144 bytes`
@@ -76,56 +109,3 @@ kubectl -n actions-runner-system wait deploy/actions-runner-controller --for con
 
 # Adhocly wait for some time until actions-runner-controller's admission webhook gets ready
 sleep 20
-
-RUNNER_LABEL=${RUNNER_LABEL:-self-hosted}
-
-if [ -n "${TEST_REPO}" ]; then
-  if [ "${USE_RUNNERSET}" != "false" ]; then
-    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerset envsubst | kubectl apply -f -
-  else
-    echo 'Deploying runnerdeployment and hra. Set USE_RUNNERSET if you want to deploy runnerset instead.'
-    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerdeploy envsubst | kubectl apply -f -
-  fi
-else
-  echo 'Skipped deploying runnerdeployment and hra. Set TEST_REPO to "yourorg/yourrepo" to deploy.'
-fi
-
-if [ -n "${TEST_ORG}" ]; then
-  if [ "${USE_RUNNERSET}" != "false" ]; then
-    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerset envsubst | kubectl apply -f -
-  else
-    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerdeploy envsubst | kubectl apply -f -
-  fi
-
-  if [ -n "${TEST_ORG_GROUP}" ]; then
-    if [ "${USE_RUNNERSET}" != "false" ]; then
-      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerset envsubst | kubectl apply -f -
-    else
-      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerdeploy envsubst | kubectl apply -f -
-    fi
-  else
-    echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ORG_GROUP to deploy.'
-  fi
-else
-  echo 'Skipped deploying organizational runnerdeployment. Set TEST_ORG to deploy.'
-fi
-
-if [ -n "${TEST_ENTERPRISE}" ]; then
-  if [ "${USE_RUNNERSET}" != "false" ]; then
-    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerset envsubst | kubectl apply -f -
-  else
-    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerdeploy envsubst | kubectl apply -f -
-  fi
-
-  if [ -n "${TEST_ENTERPRISE_GROUP}" ]; then
-    if [ "${USE_RUNNERSET}" != "false" ]; then
-      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerset envsubst | kubectl apply -f -
-    else
-      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerdeploy envsubst | kubectl apply -f -
-    fi
-  else
-    echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ENTERPRISE_GROUP to deploy.'
-  fi
-else
-  echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ENTERPRISE to deploy.'
-fi

acceptance/deploy_runners.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -e
+
+OP=${OP:-apply}
+
+RUNNER_LABEL=${RUNNER_LABEL:-self-hosted}
+
+cat acceptance/testdata/kubernetes_container_mode.envsubst.yaml  | NAMESPACE=${RUNNER_NAMESPACE} envsubst  | kubectl apply -f -
+
+if [ -n "${TEST_REPO}" ]; then
+  if [ "${USE_RUNNERSET}" != "false" ]; then
+    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerset envsubst | kubectl ${OP} -f -
+  else
+    echo "Running ${OP} runnerdeployment and hra. Set USE_RUNNERSET if you want to deploy runnerset instead."
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerdeploy envsubst | kubectl ${OP} -f -
+  fi
+else
+  echo "Skipped ${OP} for runnerdeployment and hra. Set TEST_REPO to "yourorg/yourrepo" to deploy."
+fi
+
+if [ -n "${TEST_ORG}" ]; then
+  if [ "${USE_RUNNERSET}" != "false" ]; then
+    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerset envsubst | kubectl ${OP} -f -
+  else
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerdeploy envsubst | kubectl ${OP} -f -
+  fi
+
+  if [ -n "${TEST_ORG_GROUP}" ]; then
+    if [ "${USE_RUNNERSET}" != "false" ]; then
+      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerset envsubst | kubectl ${OP} -f -
+    else
+      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerdeploy envsubst | kubectl ${OP} -f -
+    fi
+  else
+    echo "Skipped ${OP} on enterprise runnerdeployment. Set TEST_ORG_GROUP to ${OP}."
+  fi
+else
+  echo "Skipped ${OP} on organizational runnerdeployment. Set TEST_ORG to ${OP}."
+fi
+
+if [ -n "${TEST_ENTERPRISE}" ]; then
+  if [ "${USE_RUNNERSET}" != "false" ]; then
+    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerset envsubst | kubectl ${OP} -f -
+  else
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerdeploy envsubst | kubectl ${OP} -f -
+  fi
+
+  if [ -n "${TEST_ENTERPRISE_GROUP}" ]; then
+    if [ "${USE_RUNNERSET}" != "false" ]; then
+      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerset envsubst | kubectl ${OP} -f -
+    else
+      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerdeploy envsubst | kubectl ${OP} -f -
+    fi
+  else
+    echo "Skipped ${OP} on enterprise runnerdeployment. Set TEST_ENTERPRISE_GROUP to ${OP}."
+  fi
+else
+  echo "Skipped ${OP} on enterprise runnerdeployment. Set TEST_ENTERPRISE to ${OP}."
+fi

acceptance/testdata/kubernetes_container_mode.envsubst.yaml

@@ -0,0 +1,86 @@
+# USAGE:
+#   cat acceptance/testdata/kubernetes_container_mode.envsubst.yaml  | NAMESPACE=default envsubst  | kubectl apply -f -
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: k8s-mode-runner
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "create", "delete"]
+- apiGroups: [""]
+  resources: ["pods/exec"]
+  verbs: ["get", "create"]
+- apiGroups: [""]
+  resources: ["pods/log"]
+  verbs: ["get", "list", "watch",]
+- apiGroups: ["batch"]
+  resources: ["jobs"]
+  verbs: ["get", "list", "create", "delete"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list", "create", "delete"]
+# Needed to report test success by crating a cm from within workflow job step
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["create", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: runner-status-updater
+rules:
+- apiGroups: ["actions.summerwind.dev"]
+  resources: ["runners/status"]
+  verbs: ["get", "update", "patch"]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ${RUNNER_SERVICE_ACCOUNT_NAME}
+  namespace: ${NAMESPACE}
+---
+# To verify it's working, try:
+#   kubectl auth can-i --as system:serviceaccount:default:runner get pod
+# If incomplete, workflows and jobs would fail with an error message like:
+#   Error: Error: The Service account needs the following permissions [{"group":"","verbs":["get","list","create","delete"],"resource":"pods","subresource":""},{"group":"","verbs":["get","create"],"resource":"pods","subresource":"exec"},{"group":"","verbs":["get","list","watch"],"resource":"pods","subresource":"log"},{"group":"batch","verbs":["get","list","create","delete"],"resource":"jobs","subresource":""},{"group":"","verbs":["create","delete","get","list"],"resource":"secrets","subresource":""}] on the pod resource in the 'default' namespace. Please contact your self hosted runner administrator.
+#   Error: Process completed with exit code 1.
+apiVersion: rbac.authorization.k8s.io/v1
+# This role binding allows "jane" to read pods in the "default" namespace.
+# You need to already have a Role named "pod-reader" in that namespace.
+kind: RoleBinding
+metadata:
+  name: runner-k8s-mode-runner
+  namespace: ${NAMESPACE}
+subjects:
+- kind: ServiceAccount
+  name: ${RUNNER_SERVICE_ACCOUNT_NAME}
+  namespace: ${NAMESPACE}
+roleRef:
+  kind: ClusterRole
+  name: k8s-mode-runner
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: runner-runner-stat-supdater
+  namespace: ${NAMESPACE}
+subjects:
+- kind: ServiceAccount
+  name: ${RUNNER_SERVICE_ACCOUNT_NAME}
+  namespace: ${NAMESPACE}
+roleRef:
+  kind: ClusterRole
+  name: runner-status-updater
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: org-runnerdeploy-runner-work-dir
+  labels:
+    content: org-runnerdeploy-runner-work-dir
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer

acceptance/testdata/runnerdeploy.envsubst.yaml

@@ -1,3 +1,23 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}-runner-work-dir
+  labels:
+    content: ${NAME}-runner-work-dir
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}-rootless-dind-work-dir
+  labels:
+    content: ${NAME}-rootless-dind-work-dir
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+---
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerDeployment
 metadata:
@@ -39,10 +59,68 @@ spec:
       labels:
       - "${RUNNER_LABEL}"
 
+      serviceAccountName: ${RUNNER_SERVICE_ACCOUNT_NAME}
+      terminationGracePeriodSeconds: ${RUNNER_TERMINATION_GRACE_PERIOD_SECONDS}
+
+      env:
+      - name: RUNNER_GRACEFUL_STOP_TIMEOUT
+        value: "${RUNNER_GRACEFUL_STOP_TIMEOUT}"
+      - name: ROLLING_UPDATE_PHASE
+        value: "${ROLLING_UPDATE_PHASE}"
+      - name: ARC_DOCKER_MTU_PROPAGATION
+        value: "true"
+      # https://github.com/docker/docs/issues/8663
+      - name: DOCKER_DEFAULT_ADDRESS_POOL_BASE
+        value: "172.17.0.0/12"
+      - name: DOCKER_DEFAULT_ADDRESS_POOL_SIZE
+        value: "24"
+      - name: WAIT_FOR_DOCKER_SECONDS
+        value: "3"
+
+      dockerMTU: 1400
+      dockerEnv:
+      - name: RUNNER_GRACEFUL_STOP_TIMEOUT
+        value: "${RUNNER_GRACEFUL_STOP_TIMEOUT}"
+
+      # Fix the following no space left errors with rootless-dind runners that can happen while running buildx build:
+      #   ------
+      #   > [4/5] RUN go mod download:
+      #   ------
+      #   ERROR: failed to solve: failed to prepare yxsw8lv9hqnuafzlfta244l0z: mkdir /home/runner/.local/share/docker/vfs/dir/yxsw8lv9hqnuafzlfta244l0z/usr/local/go/src/cmd/compile/internal/types2/testdata: no space left on device
+      #   Error: Process completed with exit code 1.
+      #
+      volumeMounts:
+      - name: rootless-dind-work-dir
+        # Omit the /share/docker part of the /home/runner/.local/share/docker as
+        # that part is created by dockerd.
+        mountPath: /home/runner/.local
+        readOnly: false
+      volumes:
+      - name: rootless-dind-work-dir
+        ephemeral:
+          volumeClaimTemplate:
+            spec:
+              accessModes: [ "ReadWriteOnce" ]
+              storageClassName: "${NAME}-rootless-dind-work-dir"
+              resources:
+                requests:
+                  storage: 3Gi
+
       #
       # Non-standard working directory
       #
       # workDir: "/"
+
+      # # Uncomment the below to enable the kubernetes container mode
+      # # See https://github.com/actions/actions-runner-controller#runner-with-k8s-jobs
+      containerMode: ${RUNNER_CONTAINER_MODE}
+      workVolumeClaimTemplate:
+        accessModes:
+        - ReadWriteOnce
+        storageClassName: "${NAME}-runner-work-dir"
+        resources:
+          requests:
+            storage: 10Gi
 ---
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: HorizontalRunnerAutoscaler

acceptance/testdata/runnerset.envsubst.yaml

@@ -54,6 +54,16 @@ provisioner: rancher.io/local-path
 reclaimPolicy: Retain
 volumeBindingMode: WaitForFirstConsumer
 ---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}-rootless-dind-work-dir
+  labels:
+    content: ${NAME}-rootless-dind-work-dir
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+---
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerSet
 metadata:
@@ -112,18 +122,33 @@ spec:
       labels:
         app: ${NAME}
     spec:
+      serviceAccountName: ${RUNNER_SERVICE_ACCOUNT_NAME}
+      terminationGracePeriodSeconds: ${RUNNER_TERMINATION_GRACE_PERIOD_SECONDS}
       containers:
+      # # Uncomment only when non-dind-runner / you're using docker sidecar
+      # - name: docker
+      #   # Image is required for the dind sidecar definition within RunnerSet spec
+      #   image: "docker:dind"
+      #   env:
+      #   - name: RUNNER_GRACEFUL_STOP_TIMEOUT
+      #     value: "${RUNNER_GRACEFUL_STOP_TIMEOUT}"
       - name: runner
         imagePullPolicy: IfNotPresent
         env:
+        - name: RUNNER_GRACEFUL_STOP_TIMEOUT
+          value: "${RUNNER_GRACEFUL_STOP_TIMEOUT}"
         - name: RUNNER_FEATURE_FLAG_EPHEMERAL
           value: "${RUNNER_FEATURE_FLAG_EPHEMERAL}"
         - name: GOMODCACHE
           value: "/home/runner/.cache/go-mod"
+        - name: ROLLING_UPDATE_PHASE
+          value: "${ROLLING_UPDATE_PHASE}"
         # PV-backed runner work dir
         volumeMounts:
-        - name: work
-          mountPath: /runner/_work
+        # Comment out the ephemeral work volume if you're going to test the kubernetes container mode
+        # The volume and mount with the same names will be created by workVolumeClaimTemplate and the kubernetes container mode support.
+        # - name: work
+        #   mountPath: /runner/_work
         # Cache docker image layers, in case dockerdWithinRunnerContainer=true
         - name: var-lib-docker
           mountPath: /var/lib/docker
@@ -150,30 +175,57 @@ spec:
           # https://github.com/actions/setup-go/blob/56a61c9834b4a4950dbbf4740af0b8a98c73b768/src/installer.ts#L144
           mountPath: "/opt/hostedtoolcache"
       # Valid only when dockerdWithinRunnerContainer=false
-      - name: docker
-        # PV-backed runner work dir
-        volumeMounts:
-        - name: work
-          mountPath: /runner/_work
-        # Cache docker image layers, in case dockerdWithinRunnerContainer=false
-        - name: var-lib-docker
-          mountPath: /var/lib/docker
-        # image: mumoshu/actions-runner-dind:dev
+      # - name: docker
+      #   # PV-backed runner work dir
+      #   volumeMounts:
+      #   - name: work
+      #     mountPath: /runner/_work
+      #   # Cache docker image layers, in case dockerdWithinRunnerContainer=false
+      #   - name: var-lib-docker
+      #     mountPath: /var/lib/docker
+      #   # image: mumoshu/actions-runner-dind:dev
 
-        # For buildx cache
-        - name: cache
-          mountPath: "/home/runner/.cache"
+      #   # For buildx cache
+      #   - name: cache
+      #     mountPath: "/home/runner/.cache"
+
+        # For fixing no space left error on rootless dind runner
+        - name: rootless-dind-work-dir
+          # Omit the /share/docker part of the /home/runner/.local/share/docker as
+          # that part is created by dockerd.
+          mountPath: /home/runner/.local
+          readOnly: false
+
+      # Comment out the ephemeral work volume if you're going to test the kubernetes container mode
+      # volumes:
+      # - name: work
+      #   ephemeral:
+      #     volumeClaimTemplate:
+      #       spec:
+      #         accessModes:
+      #         - ReadWriteOnce
+      #         storageClassName: "${NAME}-runner-work-dir"
+      #         resources:
+      #           requests:
+      #             storage: 10Gi
+
+      # Fix the following no space left errors with rootless-dind runners that can happen while running buildx build:
+      #   ------
+      #   > [4/5] RUN go mod download:
+      #   ------
+      #   ERROR: failed to solve: failed to prepare yxsw8lv9hqnuafzlfta244l0z: mkdir /home/runner/.local/share/docker/vfs/dir/yxsw8lv9hqnuafzlfta244l0z/usr/local/go/src/cmd/compile/internal/types2/testdata: no space left on device
+      #   Error: Process completed with exit code 1.
+      #
       volumes:
-      - name: work
+      - name: rootless-dind-work-dir
         ephemeral:
           volumeClaimTemplate:
             spec:
-              accessModes:
-              - ReadWriteOnce
-              storageClassName: "${NAME}-runner-work-dir"
+              accessModes: [ "ReadWriteOnce" ]
+              storageClassName: "${NAME}-rootless-dind-work-dir"
               resources:
                 requests:
-                  storage: 10Gi
+                  storage: 3Gi
   volumeClaimTemplates:
   - metadata:
       name: vol1
@@ -251,3 +303,10 @@ spec:
   minReplicas: ${RUNNER_MIN_REPLICAS}
   maxReplicas: 10
   scaleDownDelaySecondsAfterScaleOut: ${RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT}
+  # Comment out the whole metrics if you'd like to solely test webhook-based scaling
+  metrics:
+  - type: PercentageRunnersBusy
+    scaleUpThreshold: '0.75'
+    scaleDownThreshold: '0.25'
+    scaleUpFactor: '2'
+    scaleDownFactor: '0.5'

acceptance/values.yaml

@@ -1,6 +1,18 @@
 # Set actions-runner-controller settings for testing
 logLevel: "-4"
+imagePullSecrets: []
+image:
+  # This needs to be an empty array rather than a single-item array with empty name.
+  # Otherwise you end up with the following error on helm-upgrade:
+  #   Error: UPGRADE FAILED: failed to create patch: map: map[] does not contain declared merge key: name && failed to create patch: map: map[] does not contain declared merge key: name
+  actionsRunnerImagePullSecrets: []
+runner:
+  statusUpdateHook:
+    enabled: true
+rbac:
+  allowGrantingKubernetesContainerModePermissions: true
 githubWebhookServer:
+  imagePullSecrets: []
   logLevel: "-4"
   enabled: true
   labels: {}
@@ -21,3 +33,23 @@ githubWebhookServer:
         protocol: TCP
         name: http
         nodePort: 31000
+actionsMetricsServer:
+  imagePullSecrets: []
+  logLevel: "-4"
+  enabled: true
+  labels: {}
+  replicaCount: 1
+  secret:
+    enabled: true
+    # create: true
+    name: "actions-metrics-server"
+    ### GitHub Webhook Configuration
+    #github_webhook_secret_token: ""
+  service:
+    type: NodePort
+    ports:
+      - port: 80
+        targetPort: http
+        protocol: TCP
+        name: http
+        nodePort: 31001

adrs/0000-TEMPLATE.md

@@ -0,0 +1,18 @@
+# Title
+
+<!-- ADR titles should typically be imperative sentences. -->
+
+**Status**: (Proposed|Accepted|Rejected|Superceded|Deprecated)
+
+## Context
+
+*What is the issue or background knowledge necessary for future readers
+to understand why this ADR was written?*
+
+## Decision
+
+**What** is the change being proposed? / **How** will it be implemented?*
+
+## Consequences
+
+*What becomes easier or more difficult to do because of this change?*

apis/actions.summerwind.net/v1alpha1/horizontalrunnerautoscaler_types.go

@@ -60,6 +60,9 @@ type HorizontalRunnerAutoscalerSpec struct {
 	// The earlier a scheduled override is, the higher it is prioritized.
 	// +optional
 	ScheduledOverrides []ScheduledOverride `json:"scheduledOverrides,omitempty"`
+
+	// +optional
+	GitHubAPICredentialsFrom *GitHubAPICredentialsFrom `json:"githubAPICredentialsFrom,omitempty"`
 }
 
 type ScaleUpTrigger struct {
@@ -130,7 +133,7 @@ type ScaleTargetRef struct {
 
 type MetricSpec struct {
 	// Type is the type of metric to be used for autoscaling.
-	// The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+	// It can be TotalNumberOfQueuedAndInProgressWorkflowRuns or PercentageRunnersBusy.
 	Type string `json:"type,omitempty"`
 
 	// RepositoryNames is the list of repository names to be used for calculating the metric.
@@ -170,7 +173,7 @@ type MetricSpec struct {
 }
 
 // ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule.
-// A schedule can optionally be recurring, so that the correspoding override happens every day, week, month, or year.
+// A schedule can optionally be recurring, so that the corresponding override happens every day, week, month, or year.
 type ScheduledOverride struct {
 	// StartTime is the time at which the first override starts.
 	StartTime metav1.Time `json:"startTime"`

apis/actions.summerwind.net/v1alpha1/runner_types.go

@@ -18,8 +18,10 @@ package v1alpha1
 
 import (
 	"errors"
+	"fmt"
 
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/validation/field"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -71,6 +73,19 @@ type RunnerConfig struct {
 	VolumeSizeLimit *resource.Quantity `json:"volumeSizeLimit,omitempty"`
 	// +optional
 	VolumeStorageMedium *string `json:"volumeStorageMedium,omitempty"`
+
+	// +optional
+	ContainerMode string `json:"containerMode,omitempty"`
+
+	GitHubAPICredentialsFrom *GitHubAPICredentialsFrom `json:"githubAPICredentialsFrom,omitempty"`
+}
+
+type GitHubAPICredentialsFrom struct {
+	SecretRef SecretReference `json:"secretRef,omitempty"`
+}
+
+type SecretReference struct {
+	Name string `json:"name"`
 }
 
 // RunnerPodSpec defines the desired pod spec fields of the runner pod
@@ -135,6 +150,9 @@ type RunnerPodSpec struct {
 	// +optional
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
 
+	// +optional
+	PriorityClassName string `json:"priorityClassName,omitempty"`
+
 	// +optional
 	TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
 
@@ -152,12 +170,37 @@ type RunnerPodSpec struct {
 	// +optional
 	RuntimeClassName *string `json:"runtimeClassName,omitempty"`
 
+	// +optional
+	DnsPolicy corev1.DNSPolicy `json:"dnsPolicy,omitempty"`
+
 	// +optional
 	DnsConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"`
+
+	// +optional
+	WorkVolumeClaimTemplate *WorkVolumeClaimTemplate `json:"workVolumeClaimTemplate,omitempty"`
+}
+
+func (rs *RunnerSpec) Validate(rootPath *field.Path) field.ErrorList {
+	var (
+		errList field.ErrorList
+		err     error
+	)
+
+	err = rs.validateRepository()
+	if err != nil {
+		errList = append(errList, field.Invalid(rootPath.Child("repository"), rs.Repository, err.Error()))
+	}
+
+	err = rs.validateWorkVolumeClaimTemplate()
+	if err != nil {
+		errList = append(errList, field.Invalid(rootPath.Child("workVolumeClaimTemplate"), rs.WorkVolumeClaimTemplate, err.Error()))
+	}
+
+	return errList
 }
 
 // ValidateRepository validates repository field.
-func (rs *RunnerSpec) ValidateRepository() error {
+func (rs *RunnerSpec) validateRepository() error {
 	// Enterprise, Organization and repository are both exclusive.
 	foundCount := 0
 	if len(rs.Organization) > 0 {
@@ -179,6 +222,18 @@ func (rs *RunnerSpec) ValidateRepository() error {
 	return nil
 }
 
+func (rs *RunnerSpec) validateWorkVolumeClaimTemplate() error {
+	if rs.ContainerMode != "kubernetes" {
+		return nil
+	}
+
+	if rs.WorkVolumeClaimTemplate == nil {
+		return errors.New("Spec.ContainerMode: kubernetes must have workVolumeClaimTemplate field specified")
+	}
+
+	return rs.WorkVolumeClaimTemplate.validate()
+}
+
 // RunnerStatus defines the observed state of Runner
 type RunnerStatus struct {
 	// Turns true only if the runner pod is ready.
@@ -207,13 +262,60 @@ type RunnerStatusRegistration struct {
 	ExpiresAt    metav1.Time `json:"expiresAt"`
 }
 
+type WorkVolumeClaimTemplate struct {
+	StorageClassName string                              `json:"storageClassName"`
+	AccessModes      []corev1.PersistentVolumeAccessMode `json:"accessModes"`
+	Resources        corev1.ResourceRequirements         `json:"resources"`
+}
+
+func (w *WorkVolumeClaimTemplate) validate() error {
+	if w.AccessModes == nil || len(w.AccessModes) == 0 {
+		return errors.New("Access mode should have at least one mode specified")
+	}
+
+	for _, accessMode := range w.AccessModes {
+		switch accessMode {
+		case corev1.ReadWriteOnce, corev1.ReadWriteMany:
+		default:
+			return fmt.Errorf("Access mode %v is not supported", accessMode)
+		}
+	}
+	return nil
+}
+
+func (w *WorkVolumeClaimTemplate) V1Volume() corev1.Volume {
+	return corev1.Volume{
+		Name: "work",
+		VolumeSource: corev1.VolumeSource{
+			Ephemeral: &corev1.EphemeralVolumeSource{
+				VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{
+					Spec: corev1.PersistentVolumeClaimSpec{
+						AccessModes:      w.AccessModes,
+						StorageClassName: &w.StorageClassName,
+						Resources:        w.Resources,
+					},
+				},
+			},
+		},
+	}
+}
+
+func (w *WorkVolumeClaimTemplate) V1VolumeMount(mountPath string) corev1.VolumeMount {
+	return corev1.VolumeMount{
+		MountPath: mountPath,
+		Name:      "work",
+	}
+}
+
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:printcolumn:JSONPath=".spec.enterprise",name=Enterprise,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.organization",name=Organization,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.repository",name=Repository,type=string
+// +kubebuilder:printcolumn:JSONPath=".spec.group",name=Group,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.labels",name=Labels,type=string
 // +kubebuilder:printcolumn:JSONPath=".status.phase",name=Status,type=string
+// +kubebuilder:printcolumn:JSONPath=".status.message",name=Message,type=string
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 
 // Runner is the Schema for the runners API
@@ -235,11 +337,7 @@ func (r Runner) IsRegisterable() bool {
 	}
 
 	now := metav1.Now()
-	if r.Status.Registration.ExpiresAt.Before(&now) {
-		return false
-	}
-
-	return true
+	return !r.Status.Registration.ExpiresAt.Before(&now)
 }
 
 // +kubebuilder:object:root=true

apis/actions.summerwind.net/v1alpha1/runner_webhook.go

@@ -66,15 +66,7 @@ func (r *Runner) ValidateDelete() error {
 
 // Validate validates resource spec.
 func (r *Runner) Validate() error {
-	var (
-		errList field.ErrorList
-		err     error
-	)
-
-	err = r.Spec.ValidateRepository()
-	if err != nil {
-		errList = append(errList, field.Invalid(field.NewPath("spec", "repository"), r.Spec.Repository, err.Error()))
-	}
+	errList := r.Spec.Validate(field.NewPath("spec"))
 
 	if len(errList) > 0 {
 		return apierrors.NewInvalid(r.GroupVersionKind().GroupKind(), r.Name, errList)

apis/actions.summerwind.net/v1alpha1/runnerdeployment_types.go

@@ -33,7 +33,7 @@ type RunnerDeploymentSpec struct {
 
 	// EffectiveTime is the time the upstream controller requested to sync Replicas.
 	// It is usually populated by the webhook-based autoscaler via HRA.
-	// The value is inherited to RunnerRepicaSet(s) and used to prevent ephemeral runners from unnecessarily recreated.
+	// The value is inherited to RunnerReplicaSet(s) and used to prevent ephemeral runners from unnecessarily recreated.
 	//
 	// +optional
 	// +nullable

apis/actions.summerwind.net/v1alpha1/runnerdeployment_webhook.go

@@ -66,15 +66,7 @@ func (r *RunnerDeployment) ValidateDelete() error {
 
 // Validate validates resource spec.
 func (r *RunnerDeployment) Validate() error {
-	var (
-		errList field.ErrorList
-		err     error
-	)
-
-	err = r.Spec.Template.Spec.ValidateRepository()
-	if err != nil {
-		errList = append(errList, field.Invalid(field.NewPath("spec", "template", "spec", "repository"), r.Spec.Template.Spec.Repository, err.Error()))
-	}
+	errList := r.Spec.Template.Spec.Validate(field.NewPath("spec", "template", "spec"))
 
 	if len(errList) > 0 {
 		return apierrors.NewInvalid(r.GroupVersionKind().GroupKind(), r.Name, errList)

apis/actions.summerwind.net/v1alpha1/runnerreplicaset_webhook.go

@@ -66,15 +66,7 @@ func (r *RunnerReplicaSet) ValidateDelete() error {
 
 // Validate validates resource spec.
 func (r *RunnerReplicaSet) Validate() error {
-	var (
-		errList field.ErrorList
-		err     error
-	)
-
-	err = r.Spec.Template.Spec.ValidateRepository()
-	if err != nil {
-		errList = append(errList, field.Invalid(field.NewPath("spec", "template", "spec", "repository"), r.Spec.Template.Spec.Repository, err.Error()))
-	}
+	errList := r.Spec.Template.Spec.Validate(field.NewPath("spec", "template", "spec"))
 
 	if len(errList) > 0 {
 		return apierrors.NewInvalid(r.GroupVersionKind().GroupKind(), r.Name, errList)

apis/actions.summerwind.net/v1alpha1/runnerset_types.go

@@ -33,6 +33,12 @@ type RunnerSetSpec struct {
 	// +nullable
 	EffectiveTime *metav1.Time `json:"effectiveTime,omitempty"`
 
+	// +optional
+	ServiceAccountName string `json:"serviceAccountName,omitempty"`
+
+	// +optional
+	WorkVolumeClaimTemplate *WorkVolumeClaimTemplate `json:"workVolumeClaimTemplate,omitempty"`
+
 	appsv1.StatefulSetSpec `json:",inline"`
 }
 

apis/actions.summerwind.net/v1alpha1/zz_generated.deepcopy.go

@@ -90,6 +90,22 @@ func (in *CheckRunSpec) DeepCopy() *CheckRunSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GitHubAPICredentialsFrom) DeepCopyInto(out *GitHubAPICredentialsFrom) {
+	*out = *in
+	out.SecretRef = in.SecretRef
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitHubAPICredentialsFrom.
+func (in *GitHubAPICredentialsFrom) DeepCopy() *GitHubAPICredentialsFrom {
+	if in == nil {
+		return nil
+	}
+	out := new(GitHubAPICredentialsFrom)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GitHubEventScaleUpTriggerSpec) DeepCopyInto(out *GitHubEventScaleUpTriggerSpec) {
 	*out = *in
@@ -231,6 +247,11 @@ func (in *HorizontalRunnerAutoscalerSpec) DeepCopyInto(out *HorizontalRunnerAuto
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.GitHubAPICredentialsFrom != nil {
+		in, out := &in.GitHubAPICredentialsFrom, &out.GitHubAPICredentialsFrom
+		*out = new(GitHubAPICredentialsFrom)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerSpec.
@@ -425,6 +446,11 @@ func (in *RunnerConfig) DeepCopyInto(out *RunnerConfig) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.GitHubAPICredentialsFrom != nil {
+		in, out := &in.GitHubAPICredentialsFrom, &out.GitHubAPICredentialsFrom
+		*out = new(GitHubAPICredentialsFrom)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerConfig.
@@ -741,6 +767,11 @@ func (in *RunnerPodSpec) DeepCopyInto(out *RunnerPodSpec) {
 		*out = new(v1.PodDNSConfig)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.WorkVolumeClaimTemplate != nil {
+		in, out := &in.WorkVolumeClaimTemplate, &out.WorkVolumeClaimTemplate
+		*out = new(WorkVolumeClaimTemplate)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerPodSpec.
@@ -939,6 +970,11 @@ func (in *RunnerSetSpec) DeepCopyInto(out *RunnerSetSpec) {
 		in, out := &in.EffectiveTime, &out.EffectiveTime
 		*out = (*in).DeepCopy()
 	}
+	if in.WorkVolumeClaimTemplate != nil {
+		in, out := &in.WorkVolumeClaimTemplate, &out.WorkVolumeClaimTemplate
+		*out = new(WorkVolumeClaimTemplate)
+		(*in).DeepCopyInto(*out)
+	}
 	in.StatefulSetSpec.DeepCopyInto(&out.StatefulSetSpec)
 }
 
@@ -1126,6 +1162,42 @@ func (in *ScheduledOverride) DeepCopy() *ScheduledOverride {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SecretReference) DeepCopyInto(out *SecretReference) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference.
+func (in *SecretReference) DeepCopy() *SecretReference {
+	if in == nil {
+		return nil
+	}
+	out := new(SecretReference)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WorkVolumeClaimTemplate) DeepCopyInto(out *WorkVolumeClaimTemplate) {
+	*out = *in
+	if in.AccessModes != nil {
+		in, out := &in.AccessModes, &out.AccessModes
+		*out = make([]v1.PersistentVolumeAccessMode, len(*in))
+		copy(*out, *in)
+	}
+	in.Resources.DeepCopyInto(&out.Resources)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkVolumeClaimTemplate.
+func (in *WorkVolumeClaimTemplate) DeepCopy() *WorkVolumeClaimTemplate {
+	if in == nil {
+		return nil
+	}
+	out := new(WorkVolumeClaimTemplate)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *WorkflowJobSpec) DeepCopyInto(out *WorkflowJobSpec) {
 	*out = *in

build/version.go

@@ -0,0 +1,4 @@
+package build
+
+// This is overridden at build-time using go-build ldflags. dev is the fallback value
+var Version = "NA"

charts/actions-runner-controller/Chart.yaml

@@ -15,15 +15,15 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.19.1
+version: 0.22.0
 
 # Used as the default manager tag value when no tag property is provided in the values.yaml
-appVersion: 0.24.1
+appVersion: 0.27.0
 
-home: https://github.com/actions-runner-controller/actions-runner-controller
+home: https://github.com/actions/actions-runner-controller
 
 sources:
-  - https://github.com/actions-runner-controller/actions-runner-controller
+  - https://github.com/actions/actions-runner-controller
 
 maintainers:
   - name: actions-runner-controller

charts/actions-runner-controller/README.md

@@ -4,108 +4,148 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 
 ## Values
 
-**_The values are documented as of HEAD, to review the configuration options for your chart version ensure you view this file at the relevant [tag](https://github.com/actions-runner-controller/actions-runner-controller/tags)_**
+**_The values are documented as of HEAD, to review the configuration options for your chart version ensure you view this file at the relevant [tag](https://github.com/actions/actions-runner-controller/tags)_**
 
 > _Default values are the defaults set in the charts `values.yaml`, some properties have default configurations in the code for when the property is omitted or invalid_
 
-| Key                                                      | Description                                                                                                                | Default                                                              |
-|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------|
-| `labels`                                                 | Set labels to apply to all resources in the chart                                                                          |                                                                      |
-| `replicaCount`                                           | Set the number of controller pods                                                                                          | 1                                                                    |
-| `webhookPort`                                            | Set the containerPort for the webhook Pod                                                                                  | 9443                                                                 |
-| `syncPeriod`                                             | Set the period in which the controler reconciles the desired runners count                                                 | 10m                                                                  |
-| `enableLeaderElection`                                   | Enable election configuration                                                                                              | true                                                                 |
-| `leaderElectionId`                                       | Set the election ID for the controller group                                                                               |                                                                      |
-| `githubEnterpriseServerURL`                              | Set the URL for a self-hosted GitHub Enterprise Server                                                                     |                                                                      |
-| `githubURL`                                              | Override GitHub URL to be used for GitHub API calls                                                                        |                                                                      |
-| `githubUploadURL`                                        | Override GitHub Upload URL to be used for GitHub API calls                                                                 |                                                                      |
-| `runnerGithubURL`                                        | Override GitHub URL to be used by runners during registration                                                              |                                                                      |
-| `logLevel`                                               | Set the log level of the controller container                                                                              |                                                                      |
-| `additionalVolumes`                                      | Set additional volumes to add to the manager container                                                                     |                                                                      |
-| `additionalVolumeMounts`                                 | Set additional volume mounts to add to the manager container                                                               |                                                                      |
-| `authSecret.create`                                      | Deploy the controller auth secret                                                                                          | false                                                                |
-| `authSecret.name`                                        | Set the name of the auth secret                                                                                            | controller-manager                                                   |
-| `authSecret.annotations`                                 | Set annotations for the auth Secret                                                                                        |                                                                      |
-| `authSecret.github_app_id`                               | The ID of your GitHub App. **This can't be set at the same time as `authSecret.github_token`**                             |                                                                      |
-| `authSecret.github_app_installation_id`                  | The ID of your GitHub App installation. **This can't be set at the same time as `authSecret.github_token`**                |                                                                      |
-| `authSecret.github_app_private_key`                      | The multiline string of your GitHub App's private key. **This can't be set at the same time as `authSecret.github_token`** |                                                                      |
-| `authSecret.github_token`                                | Your chosen GitHub PAT token. **This can't be set at the same time as the `authSecret.github_app_*`**                      |                                                                      |
-| `authSecret.github_basicauth_username`                     | Username for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API                 |                                                                      |
-| `authSecret.github_basicauth_password`                     | Password for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API                 |                                                                      |
-| `dockerRegistryMirror`                                   | The default Docker Registry Mirror used by runners.                                                                        |                                                                      |
-| `hostNetwork`                                            | The "hostNetwork" of the controller container                                                                              | false                                                                |
-| `image.repository`                                       | The "repository/image" of the controller container                                                                         | summerwind/actions-runner-controller                                 |
-| `image.tag`                                              | The tag of the controller container                                                                                        |                                                                      |
-| `image.actionsRunnerRepositoryAndTag`                    | The "repository/image" of the actions runner container                                                                     | summerwind/actions-runner:latest                                     |
-| `image.actionsRunnerImagePullSecrets`                    | Optional image pull secrets to be included in the runner pod's ImagePullSecrets                                            |                                                                      |
-| `image.dindSidecarRepositoryAndTag`                      | The "repository/image" of the dind sidecar container                                                                       | docker:dind                                                          |
-| `image.pullPolicy`                                       | The pull policy of the controller image                                                                                    | IfNotPresent                                                         |
-| `metrics.serviceMonitor`                                 | Deploy serviceMonitor kind for for use with prometheus-operator CRDs                                                       | false                                                                |
-| `metrics.serviceAnnotations`                             | Set annotations for the provisioned metrics service resource                                                               |                                                                      |
-| `metrics.port`                                           | Set port of metrics service                                                                                                | 8443                                                                 |
-| `metrics.proxy.enabled`                                  | Deploy kube-rbac-proxy container in controller pod                                                                         | true                                                                 |
-| `metrics.proxy.image.repository`                         | The "repository/image" of the kube-proxy container                                                                         | quay.io/brancz/kube-rbac-proxy                                       |
-| `metrics.proxy.image.tag`                                | The tag of the kube-proxy image to use when pulling the container                                                          | v0.10.0                                                              |
-| `metrics.serviceMonitorLabels`                           | Set labels to apply to ServiceMonitor resources                                                                            |                                                                      |
-| `imagePullSecrets`                                       | Specifies the secret to be used when pulling the controller pod containers                                                 |                                                                      |
-| `fullnameOverride`                                       | Override the full resource names	                                                                                        |                                                                      |
-| `nameOverride`                                           | Override the resource name prefix	                                                                                        |                                                                      |
-| `serviceAccount.annotations`                              | Set annotations to the service account                                                                                     |                                                                      |
-| `serviceAccount.create`                                  | Deploy the controller pod under a service account                                                                          | true                                                                 |
-| `podAnnotations`                                         | Set annotations for the controller pod                                                                                     |                                                                      |
-| `podLabels`                                              | Set labels for the controller pod                                                                                          |                                                                      |
-| `serviceAccount.name`                                    | Set the name of the service account                                                                                        |                                                                      |
-| `securityContext`                                        | Set the security context for each container in the controller pod                                                          |                                                                      |
-| `podSecurityContext`                                     | Set the security context to controller pod                                                                                 |                                                                      |
-| `service.annotations`                                    | Set annotations for the provisioned webhook service resource                                                               |                                                                      |
-| `service.port`                                           | Set controller service ports                                                                                                |                                                                      |
-| `service.type`                                           | Set controller service type                                                                                               |                                                                      |
-| `topologySpreadConstraints`                              | Set the controller pod topologySpreadConstraints                                                                           |                                                                      |
-| `nodeSelector`                                           | Set the controller pod nodeSelector                                                                                        |                                                                      |
-| `resources`                                              | Set the controller pod resources                                                                                           |                                                                      |
-| `affinity`                                               | Set the controller pod affinity rules                                                                                      |                                                                      |
-| `podDisruptionBudget.enabled`                            | Enables a PDB to ensure HA of controller pods                                                                              |      false                                                           |
-| `podDisruptionBudget.minAvailable`                       | Minimum number of pods that must be available after eviction                                                               |                                                                      |
-| `podDisruptionBudget.maxUnavailable`                     | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                   |                                                                      |
-| `tolerations`                                            | Set the controller pod tolerations                                                                                         |                                                                      |
-| `env`                                                    | Set environment variables for the controller container                                                                     |                                                                      |
-| `priorityClassName`                                      | Set the controller pod priorityClassName                                                                                   |                                                                      |
-| `scope.watchNamespace`                                   | Tells the controller and the github webhook server which namespace to watch if `scope.singleNamespace` is true             | `Release.Namespace` (the default namespace of the helm chart).       |
-| `scope.singleNamespace`                                  | Limit the controller to watch a single namespace                                                                           | false                                                                |
-| `certManagerEnabled`                                     | Enable cert-manager. If disabled you must set admissionWebHooks.caBundle and create TLS secrets manually                   | true                                                                 |
-| `admissionWebHooks.caBundle`                             | Base64-encoded PEM bundle containing the CA that signed the webhook's serving certificate                                  |                                                                      |
-| `githubWebhookServer.logLevel`                           | Set the log level of the githubWebhookServer container                                                                     |                                                                      |
-| `githubWebhookServer.replicaCount`                       | Set the number of webhook server pods                                                                                      | 1                                                                    |
-| `githubWebhookServer.useRunnerGroupsVisibility`          | Enable supporting runner groups with custom visibility. This will incur in extra API calls and may blow up your budget. Currently, you also need to set `githubWebhookServer.secret.enabled` to enable this feature. | false                                                                |
-| `githubWebhookServer.syncPeriod`                         | Set the period in which the controller reconciles the resources                                                            | 10m                                                                  |
-| `githubWebhookServer.enabled`                            | Deploy the webhook server pod                                                                                              | false                                                                |
-| `githubWebhookServer.secret.enabled`                      | Passes the webhook hook secret to the github-webhook-server                                                                             | false                                                                |
-| `githubWebhookServer.secret.create`                      | Deploy the webhook hook secret                                                                                             | false                                                                |
-| `githubWebhookServer.secret.name`                        | Set the name of the webhook hook secret                                                                                    | github-webhook-server                                                |
-| `githubWebhookServer.secret.github_webhook_secret_token` | Set the webhook secret token value                                                                                         |                                                                      |
-| `githubWebhookServer.imagePullSecrets`                   | Specifies the secret to be used when pulling the githubWebhookServer pod containers                                        |                                                                      |
-| `githubWebhookServer.nameOverride`                        | Override the resource name prefix	                                                                                        |                                                                      |
-| `githubWebhookServer.fullnameOverride`                    | Override the full resource names	                                                                                        |                                                                      |
-| `githubWebhookServer.serviceAccount.create`              | Deploy the githubWebhookServer under a service account                                                                     | true                                                                 |
-| `githubWebhookServer.serviceAccount.annotations`         | Set annotations for the service account                                                                                    |                                                                      |
-| `githubWebhookServer.serviceAccount.name`                | Set the service account name                                                                                               |                                                                      |
-| `githubWebhookServer.podAnnotations`                     | Set annotations for the githubWebhookServer pod                                                                            |                                                                      |
-| `githubWebhookServer.podLabels`                          | Set labels for the githubWebhookServer pod                                                                                 |                                                                      |
-| `githubWebhookServer.podSecurityContext`                 | Set the security context to githubWebhookServer pod                                                                        |                                                                      |
-| `githubWebhookServer.securityContext`                    | Set the security context for each container in the githubWebhookServer pod                                                 |                                                                      |
-| `githubWebhookServer.resources`                          | Set the githubWebhookServer pod resources                                                                                  |                                                                      |
-| `githubWebhookServer.topologySpreadConstraints`          | Set the githubWebhookServer pod topologySpreadConstraints                                                                  |                                                                      |
-| `githubWebhookServer.nodeSelector`                       | Set the githubWebhookServer pod nodeSelector                                                                               |                                                                      |
-| `githubWebhookServer.tolerations`                        | Set the githubWebhookServer pod tolerations                                                                                |                                                                      |
-| `githubWebhookServer.affinity`                           | Set the githubWebhookServer pod affinity rules                                                                             |                                                                      |
-| `githubWebhookServer.priorityClassName`                  | Set the githubWebhookServer pod priorityClassName                                                                          |                                                                      |
-| `githubWebhookServer.service.type`                       | Set githubWebhookServer service type                                                                                       |                                                                      |
-| `githubWebhookServer.service.ports`                      | Set githubWebhookServer service ports                                                                                      | `[{"port":80, "targetPort:"http", "protocol":"TCP", "name":"http"}]` |
-| `githubWebhookServer.ingress.enabled`                    | Deploy an ingress kind for the githubWebhookServer                                                                         | false                                                                |
-| `githubWebhookServer.ingress.annotations`                | Set annotations for the ingress kind                                                                                       |                                                                      |
-| `githubWebhookServer.ingress.hosts`                      | Set hosts configuration for ingress                                                                                        | `[{"host": "chart-example.local", "paths": []}]`                     |
-| `githubWebhookServer.ingress.tls`                        | Set tls configuration for ingress                                                                                          |                                                                      |
-| `githubWebhookServer.ingress.ingressClassName`           | Set ingress class name                                                                                                     |                                                                      |
-| `githubWebhookServer.podDisruptionBudget.enabled`        | Enables a PDB to ensure HA of githubwebhook pods                                                                           |      false                                                           |
-| `githubWebhookServer.podDisruptionBudget.minAvailable`   | Minimum number of pods that must be available after eviction                                                               |                                                                      |
-| `githubWebhookServer.podDisruptionBudget.maxUnavailable` | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                   |                                                                      |
+| Key                                                      | Description                                                                                                                               | Default                                                                                         |
+|----------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------|
+| `labels`                                                 | Set labels to apply to all resources in the chart                                                                                         |                                                                                                 |
+| `replicaCount`                                           | Set the number of controller pods                                                                                                         | 1                                                                                               |
+| `webhookPort`                                            | Set the containerPort for the webhook Pod                                                                                                 | 9443                                                                                            |
+| `syncPeriod`                                             | Set the period in which the controller reconciles the desired runners count                                                               | 1m                                                                                              |
+| `enableLeaderElection`                                   | Enable election configuration                                                                                                             | true                                                                                            |
+| `leaderElectionId`                                       | Set the election ID for the controller group                                                                                              |                                                                                                 |
+| `githubEnterpriseServerURL`                              | Set the URL for a self-hosted GitHub Enterprise Server                                                                                    |                                                                                                 |
+| `githubURL`                                              | Override GitHub URL to be used for GitHub API calls                                                                                       |                                                                                                 |
+| `githubUploadURL`                                        | Override GitHub Upload URL to be used for GitHub API calls                                                                                |                                                                                                 |
+| `runnerGithubURL`                                        | Override GitHub URL to be used by runners during registration                                                                             |                                                                                                 |
+| `logLevel`                                               | Set the log level of the controller container                                                                                             |                                                                                                 |
+| `logFormat`                                              | Set the log format of the controller. Valid options are "text" and "json"                                                                 | text                                                                                            |
+| `additionalVolumes`                                      | Set additional volumes to add to the manager container                                                                                    |                                                                                                 |
+| `additionalVolumeMounts`                                 | Set additional volume mounts to add to the manager container                                                                              |                                                                                                 |
+| `authSecret.create`                                      | Deploy the controller auth secret                                                                                                         | false                                                                                           |
+| `authSecret.name`                                        | Set the name of the auth secret                                                                                                           | controller-manager                                                                              |
+| `authSecret.annotations`                                 | Set annotations for the auth Secret                                                                                                       |                                                                                                 |
+| `authSecret.github_app_id`                               | The ID of your GitHub App. **This can't be set at the same time as `authSecret.github_token`**                                            |                                                                                                 |
+| `authSecret.github_app_installation_id`                  | The ID of your GitHub App installation. **This can't be set at the same time as `authSecret.github_token`**                               |                                                                                                 |
+| `authSecret.github_app_private_key`                      | The multiline string of your GitHub App's private key. **This can't be set at the same time as `authSecret.github_token`**                |                                                                                                 |
+| `authSecret.github_token`                                | Your chosen GitHub PAT token. **This can't be set at the same time as the `authSecret.github_app_*`**                                     |                                                                                                 |
+| `authSecret.github_basicauth_username`                   | Username for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API                                |                                                                                                 |
+| `authSecret.github_basicauth_password`                   | Password for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API                                |                                                                                                 |
+| `dockerRegistryMirror`                                   | The default Docker Registry Mirror used by runners.                                                                                       |                                                                                                 |
+| `hostNetwork`                                            | The "hostNetwork" of the controller container                                                                                             | false                                                                                           |
+| `image.repository`                                       | The "repository/image" of the controller container                                                                                        | summerwind/actions-runner-controller                                                            |
+| `image.tag`                                              | The tag of the controller container                                                                                                       |                                                                                                 |
+| `image.actionsRunnerRepositoryAndTag`                    | The "repository/image" of the actions runner container                                                                                    | summerwind/actions-runner:latest                                                                |
+| `image.actionsRunnerImagePullSecrets`                    | Optional image pull secrets to be included in the runner pod's ImagePullSecrets                                                           |                                                                                                 |
+| `image.dindSidecarRepositoryAndTag`                      | The "repository/image" of the dind sidecar container                                                                                      | docker:dind                                                                                     |
+| `image.pullPolicy`                                       | The pull policy of the controller image                                                                                                   | IfNotPresent                                                                                    |
+| `metrics.serviceMonitor`                                 | Deploy serviceMonitor kind for for use with prometheus-operator CRDs                                                                      | false                                                                                           |
+| `metrics.serviceAnnotations`                             | Set annotations for the provisioned metrics service resource                                                                              |                                                                                                 |
+| `metrics.port`                                           | Set port of metrics service                                                                                                               | 8443                                                                                            |
+| `metrics.proxy.enabled`                                  | Deploy kube-rbac-proxy container in controller pod                                                                                        | true                                                                                            |
+| `metrics.proxy.image.repository`                         | The "repository/image" of the kube-proxy container                                                                                        | quay.io/brancz/kube-rbac-proxy                                                                  |
+| `metrics.proxy.image.tag`                                | The tag of the kube-proxy image to use when pulling the container                                                                         | v0.10.0                                                                                         |
+| `metrics.serviceMonitorLabels`                           | Set labels to apply to ServiceMonitor resources                                                                                           |                                                                                                 |
+| `imagePullSecrets`                                       | Specifies the secret to be used when pulling the controller pod containers                                                                |                                                                                                 |
+| `fullnameOverride`                                       | Override the full resource names	                                                                                                       |                                                                                                 |
+| `nameOverride`                                           | Override the resource name prefix	                                                                                                       |                                                                                                 |
+| `serviceAccount.annotations`                             | Set annotations to the service account                                                                                                    |                                                                                                 |
+| `serviceAccount.create`                                  | Deploy the controller pod under a service account                                                                                         | true                                                                                            |
+| `podAnnotations`                                         | Set annotations for the controller pod                                                                                                    |                                                                                                 |
+| `podLabels`                                              | Set labels for the controller pod                                                                                                         |                                                                                                 |
+| `serviceAccount.name`                                    | Set the name of the service account                                                                                                       |                                                                                                 |
+| `securityContext`                                        | Set the security context for each container in the controller pod                                                                         |                                                                                                 |
+| `podSecurityContext`                                     | Set the security context to controller pod                                                                                                |                                                                                                 |
+| `service.annotations`                                    | Set annotations for the provisioned webhook service resource                                                                              |                                                                                                 |
+| `service.port`                                           | Set controller service ports                                                                                                              |                                                                                                 |
+| `service.type`                                           | Set controller service type                                                                                                               |                                                                                                 |
+| `topologySpreadConstraints`                              | Set the controller pod topologySpreadConstraints                                                                                          |                                                                                                 |
+| `nodeSelector`                                           | Set the controller pod nodeSelector                                                                                                       |                                                                                                 |
+| `resources`                                              | Set the controller pod resources                                                                                                          |                                                                                                 |
+| `affinity`                                               | Set the controller pod affinity rules                                                                                                     |                                                                                                 |
+| `podDisruptionBudget.enabled`                            | Enables a PDB to ensure HA of controller pods                                                                                             |      false                                                                                      |
+| `podDisruptionBudget.minAvailable`                       | Minimum number of pods that must be available after eviction                                                                              |                                                                                                 |
+| `podDisruptionBudget.maxUnavailable`                     | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                                  |                                                                                                 |
+| `tolerations`                                            | Set the controller pod tolerations                                                                                                        |                                                                                                 |
+| `env`                                                    | Set environment variables for the controller container                                                                                    |                                                                                                 |
+| `priorityClassName`                                      | Set the controller pod priorityClassName                                                                                                  |                                                                                                 |
+| `scope.watchNamespace`                                   | Tells the controller and the github webhook server which namespace to watch if `scope.singleNamespace` is true                            | `Release.Namespace` (the default namespace of the helm chart).                                  |
+| `scope.singleNamespace`                                  | Limit the controller to watch a single namespace                                                                                          | false                                                                                           |
+| `certManagerEnabled`                                     | Enable cert-manager. If disabled you must set admissionWebHooks.caBundle and create TLS secrets manually                                  | true                                                                                            |
+| `runner.statusUpdateHook.enabled`                        | Use custom RBAC for runners (role, role binding and service account), this will enable reporting runner statuses                          | false                                                                                           |
+| `admissionWebHooks.caBundle`                             | Base64-encoded PEM bundle containing the CA that signed the webhook's serving certificate                                                 |                                                                                                 |
+| `githubWebhookServer.logLevel`                           | Set the log level of the githubWebhookServer container                                                                                    |                                                                                                 |
+| `githubWebhookServer.logFormat`                          | Set the log format of the githubWebhookServer controller. Valid options are "text" and "json"                                             | text                                                                                            |
+| `githubWebhookServer.replicaCount`                       | Set the number of webhook server pods                                                                                                     | 1                                                                                               |
+| `githubWebhookServer.useRunnerGroupsVisibility`          | Enable supporting runner groups with custom visibility, you also need to set `githubWebhookServer.secret.enabled` to enable this feature. | false                                                                                           |
+| `githubWebhookServer.enabled`                            | Deploy the webhook server pod                                                                                                             | false                                                                                           |
+| `githubWebhookServer.queueLimit`                         | Set the queue size limit in the githubWebhookServer                                                                                       |                                                                                                 |
+| `githubWebhookServer.secret.enabled`                     | Passes the webhook hook secret to the github-webhook-server                                                                               | false                                                                                           |
+| `githubWebhookServer.secret.create`                      | Deploy the webhook hook secret                                                                                                            | false                                                                                           |
+| `githubWebhookServer.secret.name`                        | Set the name of the webhook hook secret                                                                                                   | github-webhook-server                                                                           |
+| `githubWebhookServer.secret.github_webhook_secret_token` | Set the webhook secret token value                                                                                                        |                                                                                                 |
+| `githubWebhookServer.imagePullSecrets`                   | Specifies the secret to be used when pulling the githubWebhookServer pod containers                                                       |                                                                                                 |
+| `githubWebhookServer.nameOverride`                       | Override the resource name prefix	                                                                                                       |                                                                                                 |
+| `githubWebhookServer.fullnameOverride`                   | Override the full resource names	                                                                                                       |                                                                                                 |
+| `githubWebhookServer.serviceAccount.create`              | Deploy the githubWebhookServer under a service account                                                                                    | true                                                                                            |
+| `githubWebhookServer.serviceAccount.annotations`         | Set annotations for the service account                                                                                                   |                                                                                                 |
+| `githubWebhookServer.serviceAccount.name`                | Set the service account name                                                                                                              |                                                                                                 |
+| `githubWebhookServer.podAnnotations`                     | Set annotations for the githubWebhookServer pod                                                                                           |                                                                                                 |
+| `githubWebhookServer.podLabels`                          | Set labels for the githubWebhookServer pod                                                                                                |                                                                                                 |
+| `githubWebhookServer.podSecurityContext`                 | Set the security context to githubWebhookServer pod                                                                                       |                                                                                                 |
+| `githubWebhookServer.securityContext`                    | Set the security context for each container in the githubWebhookServer pod                                                                |                                                                                                 |
+| `githubWebhookServer.resources`                          | Set the githubWebhookServer pod resources                                                                                                 |                                                                                                 |
+| `githubWebhookServer.topologySpreadConstraints`          | Set the githubWebhookServer pod topologySpreadConstraints                                                                                 |                                                                                                 |
+| `githubWebhookServer.nodeSelector`                       | Set the githubWebhookServer pod nodeSelector                                                                                              |                                                                                                 |
+| `githubWebhookServer.tolerations`                        | Set the githubWebhookServer pod tolerations                                                                                               |                                                                                                 |
+| `githubWebhookServer.affinity`                           | Set the githubWebhookServer pod affinity rules                                                                                            |                                                                                                 |
+| `githubWebhookServer.priorityClassName`                  | Set the githubWebhookServer pod priorityClassName                                                                                         |                                                                                                 |
+| `githubWebhookServer.service.type`                       | Set githubWebhookServer service type                                                                                                      |                                                                                                 |
+| `githubWebhookServer.service.ports`                      | Set githubWebhookServer service ports                                                                                                     | `[{"port":80, "targetPort:"http", "protocol":"TCP", "name":"http"}]`                            |
+| `githubWebhookServer.ingress.enabled`                    | Deploy an ingress kind for the githubWebhookServer                                                                                        | false                                                                                           |
+| `githubWebhookServer.ingress.annotations`                | Set annotations for the ingress kind                                                                                                      |                                                                                                 |
+| `githubWebhookServer.ingress.hosts`                      | Set hosts configuration for ingress                                                                                                       | `[{"host": "chart-example.local", "paths": []}]`                                                |
+| `githubWebhookServer.ingress.tls`                        | Set tls configuration for ingress                                                                                                         |                                                                                                 |
+| `githubWebhookServer.ingress.ingressClassName`           | Set ingress class name                                                                                                                    |                                                                                                 |
+| `githubWebhookServer.podDisruptionBudget.enabled`        | Enables a PDB to ensure HA of githubwebhook pods                                                                                          | false                                                                                           |
+| `githubWebhookServer.podDisruptionBudget.minAvailable`   | Minimum number of pods that must be available after eviction                                                                              |                                                                                                 |
+| `githubWebhookServer.podDisruptionBudget.maxUnavailable` | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                                  |                                                                                                 |
+| `actionsMetricsServer.logLevel`                           | Set the log level of the actionsMetricsServer container                                                                                    |                                                                                                 |
+| `actionsMetricsServer.logFormat`                          | Set the log format of the actionsMetricsServer controller. Valid options are "text" and "json"                                             | text                                                                                            |
+| `actionsMetricsServer.enabled`                            | Deploy the actions metrics server pod                                                                                                             | false                                                                                           |
+| `actionsMetricsServer.secret.enabled`                     | Passes the webhook hook secret to the github-webhook-server                                                                               | false                                                                                           |
+| `actionsMetricsServer.secret.create`                      | Deploy the webhook hook secret                                                                                                            | false                                                                                           |
+| `actionsMetricsServer.secret.name`                        | Set the name of the webhook hook secret                                                                                                   | github-webhook-server                                                                           |
+| `actionsMetricsServer.secret.github_webhook_secret_token` | Set the webhook secret token value                                                                                                        |                                                                                                 |
+| `actionsMetricsServer.imagePullSecrets`                   | Specifies the secret to be used when pulling the actionsMetricsServer pod containers                                                       |                                                                                                 |
+| `actionsMetricsServer.nameOverride`                       | Override the resource name prefix	                                                                                                       |                                                                                                 |
+| `actionsMetricsServer.fullnameOverride`                   | Override the full resource names	                                                                                                       |                                                                                                 |
+| `actionsMetricsServer.serviceAccount.create`              | Deploy the actionsMetricsServer under a service account                                                                                    | true                                                                                            |
+| `actionsMetricsServer.serviceAccount.annotations`         | Set annotations for the service account                                                                                                   |                                                                                                 |
+| `actionsMetricsServer.serviceAccount.name`                | Set the service account name                                                                                                              |                                                                                                 |
+| `actionsMetricsServer.podAnnotations`                     | Set annotations for the actionsMetricsServer pod                                                                                           |                                                                                                 |
+| `actionsMetricsServer.podLabels`                          | Set labels for the actionsMetricsServer pod                                                                                                |                                                                                                 |
+| `actionsMetricsServer.podSecurityContext`                 | Set the security context to actionsMetricsServer pod                                                                                       |                                                                                                 |
+| `actionsMetricsServer.securityContext`                    | Set the security context for each container in the actionsMetricsServer pod                                                                |                                                                                                 |
+| `actionsMetricsServer.resources`                          | Set the actionsMetricsServer pod resources                                                                                                 |                                                                                                 |
+| `actionsMetricsServer.topologySpreadConstraints`          | Set the actionsMetricsServer pod topologySpreadConstraints                                                                                 |                                                                                                 |
+| `actionsMetricsServer.nodeSelector`                       | Set the actionsMetricsServer pod nodeSelector                                                                                              |                                                                                                 |
+| `actionsMetricsServer.tolerations`                        | Set the actionsMetricsServer pod tolerations                                                                                               |                                                                                                 |
+| `actionsMetricsServer.affinity`                           | Set the actionsMetricsServer pod affinity rules                                                                                            |                                                                                                 |
+| `actionsMetricsServer.priorityClassName`                  | Set the actionsMetricsServer pod priorityClassName                                                                                         |                                                                                                 |
+| `actionsMetricsServer.service.type`                       | Set actionsMetricsServer service type                                                                                                      |                                                                                                 |
+| `actionsMetricsServer.service.ports`                      | Set actionsMetricsServer service ports                                                                                                     | `[{"port":80, "targetPort:"http", "protocol":"TCP", "name":"http"}]`                            |
+| `actionsMetricsServer.ingress.enabled`                    | Deploy an ingress kind for the actionsMetricsServer                                                                                        | false                                                                                           |
+| `actionsMetricsServer.ingress.annotations`                | Set annotations for the ingress kind                                                                                                      |                                                                                                 |
+| `actionsMetricsServer.ingress.hosts`                      | Set hosts configuration for ingress                                                                                                       | `[{"host": "chart-example.local", "paths": []}]`                                                |
+| `actionsMetricsServer.ingress.tls`                        | Set tls configuration for ingress                                                                                                         |                                                                                                 |
+| `actionsMetricsServer.ingress.ingressClassName`           | Set ingress class name                                                                                                                    |                                                                                                 |
+| `actionsMetrics.serviceMonitor`                                 | Deploy serviceMonitor kind for for use with prometheus-operator CRDs                                                                      | false                                                                                           |
+| `actionsMetrics.serviceAnnotations`                             | Set annotations for the provisioned actions metrics service resource                                                                              |                                                                                                 |
+| `actionsMetrics.port`                                           | Set port of actions metrics service                                                                                                               | 8443                                                                                            |
+| `actionsMetrics.proxy.enabled`                                  | Deploy kube-rbac-proxy container in controller pod                                                                                        | true                                                                                            |
+| `actionsMetrics.proxy.image.repository`                         | The "repository/image" of the kube-proxy container                                                                                        | quay.io/brancz/kube-rbac-proxy                                                                  |
+| `actionsMetrics.proxy.image.tag`                                | The tag of the kube-proxy image to use when pulling the container                                                                         | v0.10.0                                                                                         |
+| `actionsMetrics.serviceMonitorLabels`                           | Set labels to apply to ServiceMonitor resources                                                                                           |                                                                                                 |

charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -61,6 +61,16 @@ spec:
                         type: integer
                     type: object
                   type: array
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 maxReplicas:
                   description: MaxReplicas is the maximum number of replicas the deployment is allowed to scale
                   type: integer
@@ -92,7 +102,7 @@ spec:
                         description: ScaleUpThreshold is the percentage of busy runners greater than which will trigger the hpa to scale runners up.
                         type: string
                       type:
-                        description: Type is the type of metric to be used for autoscaling. The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+                        description: Type is the type of metric to be used for autoscaling. It can be TotalNumberOfQueuedAndInProgressWorkflowRuns or PercentageRunnersBusy.
                         type: string
                     type: object
                   type: array
@@ -170,7 +180,7 @@ spec:
                 scheduledOverrides:
                   description: ScheduledOverrides is the list of ScheduledOverride. It can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. The earlier a scheduled override is, the higher it is prioritized.
                   items:
-                    description: ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally be recurring, so that the correspoding override happens every day, week, month, or year.
+                    description: ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally be recurring, so that the corresponding override happens every day, week, month, or year.
                     properties:
                       endTime:
                         description: EndTime is the time at which the first override ends.

charts/actions-runner-controller/docs/UPGRADING.md

@@ -24,7 +24,7 @@ Due to the above you can't just do a `helm upgrade` to release the latest versio
 # REMEMBER TO UPDATE THE CHART_VERSION TO RELEVANT CHART VERISON!!!!
 CHART_VERSION=0.18.0
 
-curl -L https://github.com/actions-runner-controller/actions-runner-controller/releases/download/actions-runner-controller-${CHART_VERSION}/actions-runner-controller-${CHART_VERSION}.tgz | tar zxv --strip 1 actions-runner-controller/crds
+curl -L https://github.com/actions/actions-runner-controller/releases/download/actions-runner-controller-${CHART_VERSION}/actions-runner-controller-${CHART_VERSION}.tgz | tar zxv --strip 1 actions-runner-controller/crds
 
 kubectl replace -f crds/
 ```

charts/actions-runner-controller/templates/_actions_metrics_server_helpers.tpl

@@ -0,0 +1,60 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "actions-runner-controller-actions-metrics-server.name" -}}
+{{- default .Chart.Name .Values.actionsMetricsServer.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{- define "actions-runner-controller-actions-metrics-server.instance" -}}
+{{- printf "%s-%s" .Release.Name "actions-metrics-server" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "actions-runner-controller-actions-metrics-server.fullname" -}}
+{{- if .Values.actionsMetricsServer.fullnameOverride }}
+{{- .Values.actionsMetricsServer.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.actionsMetricsServer.nameOverride }}
+{{- $instance := include "actions-runner-controller-actions-metrics-server.instance" . }}
+{{- if contains $name $instance }}
+{{- $instance | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s-%s" .Release.Name $name "actions-metrics-server" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "actions-runner-controller-actions-metrics-server.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "actions-runner-controller-actions-metrics-server.name" . }}
+app.kubernetes.io/instance: {{ include "actions-runner-controller-actions-metrics-server.instance" . }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "actions-runner-controller-actions-metrics-server.serviceAccountName" -}}
+{{- if .Values.actionsMetricsServer.serviceAccount.create }}
+{{- default (include "actions-runner-controller-actions-metrics-server.fullname" .) .Values.actionsMetricsServer.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.actionsMetricsServer.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{- define "actions-runner-controller-actions-metrics-server.secretName" -}}
+{{- default (include "actions-runner-controller-actions-metrics-server.fullname" .) .Values.actionsMetricsServer.secret.name }}
+{{- end }}
+
+{{- define "actions-runner-controller-actions-metrics-server.roleName" -}}
+{{- include "actions-runner-controller-actions-metrics-server.fullname" . }}
+{{- end }}
+
+{{- define "actions-runner-controller-actions-metrics-server.serviceMonitorName" -}}
+{{- include "actions-runner-controller-actions-metrics-server.fullname" . | trunc 47 }}-service-monitor
+{{- end }}

charts/actions-runner-controller/templates/_helpers.tpl

@@ -114,4 +114,4 @@ Create the name of the service account to use
 
 {{- define "actions-runner-controller.pdbName" -}}
 {{- include "actions-runner-controller.fullname" . | trunc 59 }}-pdb
-{{- end }}
+{{- end }}

charts/actions-runner-controller/templates/actionsmetrics.deployment.yaml

@@ -0,0 +1,162 @@
+{{- if .Values.actionsMetricsServer.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "actions-runner-controller-actions-metrics-server.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.actionsMetricsServer.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "actions-runner-controller-actions-metrics-server.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.actionsMetricsServer.podAnnotations }}
+      annotations:
+        kubectl.kubernetes.io/default-logs-container: "github-webhook-server"
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "actions-runner-controller-actions-metrics-server.selectorLabels" . | nindent 8 }}
+      {{- with .Values.actionsMetricsServer.podLabels }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      {{- with .Values.actionsMetricsServer.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "actions-runner-controller-actions-metrics-server.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.actionsMetricsServer.podSecurityContext | nindent 8 }}
+      {{- with .Values.actionsMetricsServer.priorityClassName }}
+      priorityClassName: "{{ . }}"
+      {{- end }}
+      containers:
+      - args:
+        {{- $metricsHost := .Values.metrics.proxy.enabled | ternary "127.0.0.1" "0.0.0.0" }}
+        {{- $metricsPort := .Values.metrics.proxy.enabled | ternary "8080" .Values.metrics.port }}
+        - "--metrics-addr={{ $metricsHost }}:{{ $metricsPort }}"
+        {{- if .Values.actionsMetricsServer.logLevel }}
+        - "--log-level={{ .Values.actionsMetricsServer.logLevel }}"
+        {{- end }}
+        {{- if .Values.runnerGithubURL  }}
+        - "--runner-github-url={{ .Values.runnerGithubURL }}"
+        {{- end }}
+        {{- if .Values.actionsMetricsServer.logFormat  }}  
+        - "--log-format={{ .Values.actionsMetricsServer.logFormat }}"
+        {{- end }}
+        command:
+        - "/actions-metrics-server"
+        env:
+        - name: GITHUB_WEBHOOK_SECRET_TOKEN
+          valueFrom:
+            secretKeyRef:
+              key: github_webhook_secret_token
+              name: {{ include "actions-runner-controller-actions-metrics-server.secretName" . }}
+              optional: true
+        {{- if .Values.githubEnterpriseServerURL  }}
+        - name: GITHUB_ENTERPRISE_URL
+          value: {{ .Values.githubEnterpriseServerURL }}
+        {{- end }}
+        {{- if .Values.githubURL  }}
+        - name: GITHUB_URL
+          value: {{ .Values.githubURL }}
+        {{- end }}
+        {{- if .Values.githubUploadURL  }}
+        - name: GITHUB_UPLOAD_URL
+          value: {{ .Values.githubUploadURL }}
+        {{- end }}
+        {{- if .Values.actionsMetricsServer.secret.enabled }}
+        - name: GITHUB_TOKEN
+          valueFrom:
+            secretKeyRef:
+              key: github_token
+              name: {{ include "actions-runner-controller.githubWebhookServerSecretName" . }}
+              optional: true
+        - name: GITHUB_APP_ID
+          valueFrom:
+            secretKeyRef:
+              key: github_app_id
+              name: {{ include "actions-runner-controller.githubWebhookServerSecretName" . }}
+              optional: true
+        - name: GITHUB_APP_INSTALLATION_ID
+          valueFrom:
+            secretKeyRef:
+              key: github_app_installation_id
+              name: {{ include "actions-runner-controller.githubWebhookServerSecretName" . }}
+              optional: true
+        - name: GITHUB_APP_PRIVATE_KEY
+          valueFrom:
+            secretKeyRef:
+              key: github_app_private_key
+              name: {{ include "actions-runner-controller.githubWebhookServerSecretName" . }}
+              optional: true
+        {{- if .Values.authSecret.github_basicauth_username }}
+        - name: GITHUB_BASICAUTH_USERNAME
+          value: {{ .Values.authSecret.github_basicauth_username }}
+        {{- end }}
+        - name: GITHUB_BASICAUTH_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              key: github_basicauth_password
+              name: {{ include "actions-runner-controller.secretName" . }}
+              optional: true
+        {{- end }}
+        {{- range $key, $val := .Values.actionsMetricsServer.env }}
+        - name: {{ $key }}
+          value: {{ $val | quote }}
+        {{- end }}
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag  | default (cat "v" .Chart.AppVersion | replace " " "") }}"
+        name: actions-metrics-server
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        ports:
+        - containerPort: 8000
+          name: http
+          protocol: TCP
+        {{- if not .Values.metrics.proxy.enabled }}
+        - containerPort: {{ .Values.metrics.port }}
+          name: metrics-port
+          protocol: TCP
+        {{- end }}
+        resources:
+          {{- toYaml .Values.actionsMetricsServer.resources | nindent 12 }}
+        securityContext:
+          {{- toYaml .Values.actionsMetricsServer.securityContext | nindent 12 }}
+      {{- if .Values.metrics.proxy.enabled }}
+      - args:
+        - "--secure-listen-address=0.0.0.0:{{ .Values.metrics.port }}"
+        - "--upstream=http://127.0.0.1:8080/"
+        - "--logtostderr=true"
+        - "--v=10"
+        image: "{{ .Values.metrics.proxy.image.repository }}:{{ .Values.metrics.proxy.image.tag }}"
+        name: kube-rbac-proxy
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        ports:
+        - containerPort: {{ .Values.metrics.port }}
+          name: metrics-port
+        resources:
+          {{- toYaml .Values.resources | nindent 12 }}
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 12 }}
+      {{- end }}
+      terminationGracePeriodSeconds: 10
+      {{- with .Values.actionsMetricsServer.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.actionsMetricsServer.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.actionsMetricsServer.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.actionsMetricsServer.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/actions-runner-controller/templates/actionsmetrics.ingress.yaml.yml

@@ -0,0 +1,47 @@
+{{- if .Values.actionsMetricsServer.ingress.enabled -}}
+{{- $fullName := include "actions-runner-controller-actions-metrics-server.fullname" . -}}
+{{- $svcPort := (index .Values.actionsMetricsServer.service.ports 0).port -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ $fullName }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+  {{- with .Values.actionsMetricsServer.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.actionsMetricsServer.ingress.tls }}
+  tls:
+    {{- range .Values.actionsMetricsServer.ingress.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  {{- with .Values.actionsMetricsServer.ingress.ingressClassName }}
+  ingressClassName: {{ . }}
+  {{- end }}
+  rules:
+    {{- range .Values.actionsMetricsServer.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- if .extraPaths }}
+          {{- toYaml .extraPaths | nindent 10 }}
+          {{- end }}
+          {{- range .paths }}
+          - path: {{ .path }}
+            pathType: {{ .pathType }}
+            backend:
+              service:
+               name: {{ $fullName }}
+               port:
+                 number: {{ $svcPort }}
+          {{- end }}
+    {{- end }}
+  {{- end }}

charts/actions-runner-controller/templates/actionsmetrics.service.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.actionsMetricsServer.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "actions-runner-controller-actions-metrics-server.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+{{- if .Values.actionsMetricsServer.service.annotations }}
+  annotations:
+    {{ toYaml .Values.actionsMetricsServer.service.annotations | nindent 4 }}
+{{- end }}
+spec:
+  type: {{ .Values.actionsMetricsServer.service.type }}
+  ports:
+    {{ range $_, $port := .Values.actionsMetricsServer.service.ports -}}
+    - {{ $port | toYaml | nindent 6 }}
+    {{- end }}
+    {{- if .Values.metrics.serviceMonitor }}
+    - name: metrics-port
+      port: {{ .Values.metrics.port }}
+      targetPort: metrics-port
+    {{- end }}
+  selector:
+    {{- include "actions-runner-controller-actions-metrics-server.selectorLabels" . | nindent 4 }}
+{{- end }}

charts/actions-runner-controller/templates/actionsmetrics.serviceaccount.yaml.yml

@@ -0,0 +1,15 @@
+{{- if .Values.actionsMetricsServer.enabled -}}
+{{- if .Values.actionsMetricsServer.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "actions-runner-controller-actions-metrics-server.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+  {{- with .Values.actionsMetricsServer.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}
+{{- end }}

charts/actions-runner-controller/templates/actionsmetrics.servicemonitor.yaml.yml

@@ -0,0 +1,25 @@
+{{- if and .Values.actionsMetricsServer.enabled .Values.actionsMetrics.serviceMonitor }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+  {{- with .Values.actionsMetricsServer.serviceMonitorLabels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  name: {{ include "actions-runner-controller-actions-metrics-server.serviceMonitorName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  endpoints:
+    - path: /metrics
+      port: metrics-port
+      {{- if .Values.actionsMetrics.proxy.enabled }}
+      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      scheme: https
+      tlsConfig:
+        insecureSkipVerify: true
+      {{- end }}
+  selector:
+    matchLabels:
+      {{- include "actions-runner-controller-actions-metrics-server.selectorLabels" . | nindent 6 }}
+{{- end }}

charts/actions-runner-controller/templates/controller.metrics.serviceMonitor.yaml

@@ -8,6 +8,7 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
   name: {{ include "actions-runner-controller.serviceMonitorName" . }}
+  namespace: {{ .Release.Namespace }}
 spec:
   endpoints:
     - path: /metrics

charts/actions-runner-controller/templates/controller.pdb.yaml

@@ -1,5 +1,5 @@
 {{- if .Values.podDisruptionBudget.enabled }}
-apiVersion: policy/v1beta1
+apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
   labels:

charts/actions-runner-controller/templates/deployment.yaml

@@ -58,15 +58,18 @@ spec:
         {{- if .Values.scope.singleNamespace }}
         - "--watch-namespace={{ default .Release.Namespace .Values.scope.watchNamespace }}"
         {{- end }}
-        {{- if .Values.githubAPICacheDuration }}
-        - "--github-api-cache-duration={{ .Values.githubAPICacheDuration }}"
-        {{- end }}
         {{- if .Values.logLevel }}
         - "--log-level={{ .Values.logLevel }}"
         {{- end }}
         {{- if .Values.runnerGithubURL  }}
         - "--runner-github-url={{ .Values.runnerGithubURL }}"
         {{- end }}
+        {{- if .Values.runner.statusUpdateHook.enabled }}
+        - "--runner-status-update-hook"
+        {{- end }}
+        {{- if .Values.logFormat  }}  
+        - "--log-format={{ .Values.logFormat }}"
+        {{- end }}
         command:
         - "/manager"
         env:
@@ -118,10 +121,14 @@ spec:
               name: {{ include "actions-runner-controller.secretName" . }}
               optional: true
         {{- end }}
+        {{- if kindIs "slice" .Values.env }}
+        {{- toYaml .Values.env | nindent 8 }}
+        {{- else }}
         {{- range $key, $val := .Values.env }}
         - name: {{ $key }}
           value: {{ $val | quote }}
         {{- end }}
+        {{- end }}
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (cat "v" .Chart.AppVersion | replace " " "") }}"
         name: manager
         imagePullPolicy: {{ .Values.image.pullPolicy }}

charts/actions-runner-controller/templates/githubwebhook.deployment.yaml

@@ -39,7 +39,6 @@ spec:
         {{- $metricsHost := .Values.metrics.proxy.enabled | ternary "127.0.0.1" "0.0.0.0" }}
         {{- $metricsPort := .Values.metrics.proxy.enabled | ternary "8080" .Values.metrics.port }}
         - "--metrics-addr={{ $metricsHost }}:{{ $metricsPort }}"
-        - "--sync-period={{ .Values.githubWebhookServer.syncPeriod }}"
         {{- if .Values.githubWebhookServer.logLevel }}
         - "--log-level={{ .Values.githubWebhookServer.logLevel }}"
         {{- end }}
@@ -49,6 +48,12 @@ spec:
         {{- if .Values.runnerGithubURL  }}
         - "--runner-github-url={{ .Values.runnerGithubURL }}"
         {{- end }}
+        {{- if .Values.githubWebhookServer.queueLimit }}
+        - "--queue-limit={{ .Values.githubWebhookServer.queueLimit }}"
+        {{- end }}
+        {{- if .Values.githubWebhookServer.logFormat  }}  
+        - "--log-format={{ .Values.githubWebhookServer.logFormat }}"
+        {{- end }}
         command:
         - "/github-webhook-server"
         env:

charts/actions-runner-controller/templates/githubwebhook.ingress.yaml

@@ -1,13 +1,7 @@
 {{- if .Values.githubWebhookServer.ingress.enabled -}}
 {{- $fullName := include "actions-runner-controller-github-webhook-server.fullname" . -}}
 {{- $svcPort := (index .Values.githubWebhookServer.service.ports 0).port -}}
-{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
 apiVersion: networking.k8s.io/v1
-{{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
-apiVersion: networking.k8s.io/v1beta1
-{{- else if .Capabilities.APIVersions.Has "extensions/v1beta1" }}
-apiVersion: extensions/v1beta1
-{{- end }}
 kind: Ingress
 metadata:
   name: {{ $fullName }}
@@ -42,19 +36,12 @@ spec:
           {{- end }}
           {{- range .paths }}
           - path: {{ .path }}
-            {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
             pathType: {{ .pathType }}
-            {{- end }}
             backend:
-              {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
               service:
                name: {{ $fullName }}
                port:
                  number: {{ $svcPort }}
-              {{- else }}
-              serviceName: {{ $fullName }}
-              servicePort: {{ $svcPort }}
-              {{- end }}
           {{- end }}
     {{- end }}
   {{- end }}

charts/actions-runner-controller/templates/githubwebhook.pdb.yaml

@@ -1,5 +1,5 @@
 {{- if .Values.githubWebhookServer.podDisruptionBudget.enabled }}
-apiVersion: policy/v1beta1
+apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
   labels:

charts/actions-runner-controller/templates/githubwebhook.secrets.yaml

@@ -12,5 +12,17 @@ data:
 {{- if .Values.githubWebhookServer.secret.github_webhook_secret_token }}
   github_webhook_secret_token: {{ .Values.githubWebhookServer.secret.github_webhook_secret_token | toString | b64enc }}
 {{- end }}
+{{- if .Values.githubWebhookServer.secret.github_app_id }}
+  github_app_id: {{ .Values.githubWebhookServer.secret.github_app_id | toString | b64enc }}
+{{- end }}
+{{- if .Values.githubWebhookServer.secret.github_app_installation_id }}
+  github_app_installation_id: {{ .Values.githubWebhookServer.secret.github_app_installation_id | toString | b64enc }}
+{{- end }}
+{{- if .Values.githubWebhookServer.secret.github_app_private_key }}
+  github_app_private_key: {{ .Values.githubWebhookServer.secret.github_app_private_key | toString | b64enc }}
+{{- end }}
+{{- if .Values.githubWebhookServer.secret.github_token }}
+  github_token: {{ .Values.githubWebhookServer.secret.github_token | toString | b64enc }}
+{{- end }}
 {{- end }}
 {{- end }}

charts/actions-runner-controller/templates/githubwebhook.serviceMonitor.yaml

@@ -8,6 +8,7 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
   name: {{ include "actions-runner-controller-github-webhook-server.serviceMonitorName" . }}
+  namespace: {{ .Release.Namespace }}
 spec:
   endpoints:
     - path: /metrics

charts/actions-runner-controller/templates/manager_role.yaml

@@ -250,3 +250,72 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+{{- if .Values.runner.statusUpdateHook.enabled }}
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - create
+  - delete
+  - get
+{{- end }}
+{{- if .Values.rbac.allowGrantingKubernetesContainerModePermissions }}
+{{/* These permissions are required by ARC to create RBAC resources for the runner pod to use the kubernetes container mode. */}}
+{{/* See https://github.com/actions/actions-runner-controller/pull/1268/files#r917331632 */}}
+- apiGroups:
+  - ""
+  resources:
+  - pods/exec
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/log
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - "batch"
+  resources:
+  - jobs
+  verbs:
+  - get
+  - list
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - delete
+{{- end }}

charts/actions-runner-controller/templates/webhook_configs.yaml

@@ -1,4 +1,8 @@
-
+{{/*
+We will use a self managed CA if one is not provided by cert-manager
+*/}}
+{{- $ca := genCA "actions-runner-ca" 3650 }}
+{{- $cert := genSignedCert (printf "%s.%s.svc" (include "actions-runner-controller.webhookServiceName" .) .Release.Namespace) nil (list (printf "%s.%s.svc" (include "actions-runner-controller.webhookServiceName" .) .Release.Namespace)) 3650 $ca }}
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
@@ -20,6 +24,8 @@ webhooks:
   clientConfig:
     {{- if .Values.admissionWebHooks.caBundle }}
     caBundle: {{ quote .Values.admissionWebHooks.caBundle }}
+    {{- else if not .Values.certManagerEnabled }}
+    caBundle: {{ $ca.Cert | b64enc | quote }}
     {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
@@ -47,7 +53,9 @@ webhooks:
   {{- end }}
   clientConfig:
     {{- if .Values.admissionWebHooks.caBundle }}
-    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ quote .Values.admissionWebHooks.caBundle }}
+    {{- else if not .Values.certManagerEnabled }}
+    caBundle: {{ $ca.Cert | b64enc | quote }}
     {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
@@ -75,7 +83,9 @@ webhooks:
   {{- end }}
   clientConfig:
     {{- if .Values.admissionWebHooks.caBundle }}
-    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ quote .Values.admissionWebHooks.caBundle }}
+    {{- else if not .Values.certManagerEnabled }}
+    caBundle: {{ $ca.Cert | b64enc | quote }}
     {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
@@ -103,7 +113,9 @@ webhooks:
   {{- end }}
   clientConfig:
     {{- if .Values.admissionWebHooks.caBundle }}
-    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ quote .Values.admissionWebHooks.caBundle }}
+    {{- else if not .Values.certManagerEnabled }}
+    caBundle: {{ $ca.Cert | b64enc | quote }}
     {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
@@ -144,7 +156,9 @@ webhooks:
   {{- end }}
   clientConfig:
     {{- if .Values.admissionWebHooks.caBundle }}
-    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ quote .Values.admissionWebHooks.caBundle }}
+    {{- else if not .Values.certManagerEnabled }}
+    caBundle: {{ $ca.Cert | b64enc | quote }}
     {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
@@ -172,7 +186,9 @@ webhooks:
   {{- end }}
   clientConfig:
     {{- if .Values.admissionWebHooks.caBundle }}
-    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ quote .Values.admissionWebHooks.caBundle }}
+    {{- else if not .Values.certManagerEnabled }}
+    caBundle: {{ $ca.Cert | b64enc | quote }}
     {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
@@ -200,7 +216,9 @@ webhooks:
   {{- end }}
   clientConfig:
     {{- if .Values.admissionWebHooks.caBundle }}
-    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ quote .Values.admissionWebHooks.caBundle }}
+    {{- else if not .Values.certManagerEnabled }}
+    caBundle: {{ $ca.Cert | b64enc | quote }}
     {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
@@ -219,3 +237,18 @@ webhooks:
     resources:
     - runnerreplicasets
   sideEffects: None
+{{ if not (or (hasKey .Values.admissionWebHooks "caBundle") .Values.certManagerEnabled) }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "actions-runner-controller.servingCertName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+type: kubernetes.io/tls
+data:
+  tls.crt: {{ $cert.Cert | b64enc | quote }}
+  tls.key: {{ $cert.Key | b64enc | quote }}
+  ca.crt: {{ $ca.Cert | b64enc | quote }}
+{{- end }}

charts/actions-runner-controller/values.yaml

@@ -15,12 +15,6 @@ enableLeaderElection: true
 # Must be unique if more than one controller installed onto the same namespace.
 #leaderElectionId: "actions-runner-controller"
 
-# DEPRECATED: This has been removed as unnecessary in #1192
-# The controller tries its best not to repeat the duplicate GitHub API call
-# within this duration.
-# Defaults to syncPeriod - 10s.
-#githubAPICacheDuration: 30s
-
 # The URL of your GitHub Enterprise server, if you're using one.
 #githubEnterpriseServerURL: https://github.example.com
 
@@ -36,7 +30,7 @@ enableLeaderElection: true
 #
 # Do set authSecret.enabled=false and set env if you want full control over
 # the GitHub authn related envvars of the container.
-# See https://github.com/actions-runner-controller/actions-runner-controller/pull/937 for more details.
+# See https://github.com/actions/actions-runner-controller/pull/937 for more details.
 authSecret:
   enabled: true
   create: false
@@ -67,6 +61,18 @@ imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
 
+runner:
+  statusUpdateHook:
+    enabled: false
+
+rbac:
+  {}
+  # # This allows ARC to dynamically create a ServiceAccount and a Role for each Runner pod that uses "kubernetes" container mode,
+  # # by extending ARC's manager role to have the same permissions required by the pod runs the runner agent in "kubernetes" container mode.
+  # # Without this, Kubernetes blocks ARC to create the role to prevent a priviledge escalation.
+  # # See https://github.com/actions/actions-runner-controller/pull/1268/files#r917327010
+  # allowGrantingKubernetesContainerModePermissions: true
+
 serviceAccount:
   # Specifies whether a service account should be created
   create: true
@@ -109,7 +115,7 @@ metrics:
     enabled: true
     image:
       repository: quay.io/brancz/kube-rbac-proxy
-      tag: v0.12.0
+      tag: v0.13.1
 
 resources:
   {}
@@ -143,10 +149,20 @@ priorityClassName: ""
 
 env:
   {}
+  # specify additional environment variables for the controller pod.
+  # It's possible to specify either key vale pairs e.g.:
   # http_proxy: "proxy.com:8080"
   # https_proxy: "proxy.com:8080"
   # no_proxy: ""
 
+  # or a list of complete environment variable definitions e.g.:
+  # - name: GITHUB_APP_INSTALLATION_ID
+  #   valueFrom:
+  #     secretKeyRef:
+  #       key: some_key_in_the_secret
+  #       name: some-secret-name
+  #       optional: true
+
 ## specify additional volumes to mount in the manager container, this can be used
 ## to specify additional storage of material or to inject files from ConfigMaps
 ## into the running container
@@ -169,20 +185,31 @@ admissionWebHooks:
   #caBundle: "Ci0tLS0tQk...<base64-encoded PEM bundle containing the CA that signed the webhook's serving certificate>...tLS0K"
 
 # There may be alternatives to setting `hostNetwork: true`, see
-# https://github.com/actions-runner-controller/actions-runner-controller/issues/1005#issuecomment-993097155
+# https://github.com/actions/actions-runner-controller/issues/1005#issuecomment-993097155
 #hostNetwork: true
 
+## specify log format for actions runner controller.  Valid options are "text" and "json"
+logFormat: text
+
 githubWebhookServer:
   enabled: false
   replicaCount: 1
-  syncPeriod: 10m
   useRunnerGroupsVisibility: false
+  ## specify log format for github webhook controller.  Valid options are "text" and "json"
+  logFormat: text
   secret:
     enabled: false
     create: false
     name: "github-webhook-server"
     ### GitHub Webhook Configuration
     github_webhook_secret_token: ""
+    ### GitHub Apps Configuration
+    ## NOTE: IDs MUST be strings, use quotes
+    #github_app_id: ""
+    #github_app_installation_id: ""
+    #github_app_private_key: |
+    ### GitHub PAT Configuration
+    #github_token: ""
   imagePullSecrets: []
   nameOverride: ""
   fullnameOverride: ""
@@ -248,3 +275,101 @@ githubWebhookServer:
     enabled: false
     # minAvailable: 1
     # maxUnavailable: 3
+  # queueLimit: 100
+
+actionsMetrics:
+  serviceAnnotations: {}
+  # Set serviceMonitor=true to create a service monitor
+  # as a part of the helm release.
+  # Do note that you also need actionsMetricsServer.enabled=true
+  # to deploy the actions-metrics-server whose k8s service is referenced by the service monitor.
+  serviceMonitor: false
+  serviceMonitorLabels: {}
+  port: 8443
+  proxy:
+    enabled: true
+    image:
+      repository: quay.io/brancz/kube-rbac-proxy
+      tag: v0.13.1
+
+actionsMetricsServer:
+  enabled: false
+  # DO NOT CHANGE THIS!
+  # See the thread below for more context.
+  # https://github.com/actions/actions-runner-controller/pull/1814#discussion_r974758924
+  replicaCount: 1
+  ## specify log format for github webhook controller.  Valid options are "text" and "json"
+  logFormat: text
+  secret:
+    enabled: false
+    create: false
+    name: "actions-metrics-server"
+    ### GitHub Webhook Configuration
+    github_webhook_secret_token: ""
+    ### GitHub Apps Configuration
+    ## NOTE: IDs MUST be strings, use quotes
+    #github_app_id: ""
+    #github_app_installation_id: ""
+    #github_app_private_key: |
+    ### GitHub PAT Configuration
+    #github_token: ""
+  imagePullSecrets: []
+  nameOverride: ""
+  fullnameOverride: ""
+  serviceAccount:
+    # Specifies whether a service account should be created
+    create: true
+    # Annotations to add to the service account
+    annotations: {}
+    # The name of the service account to use.
+    # If not set and create is true, a name is generated using the fullname template
+    name: ""
+  podAnnotations: {}
+  podLabels: {}
+  podSecurityContext: {}
+  # fsGroup: 2000
+  securityContext: {}
+  resources: {}
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+  priorityClassName: ""
+  service:
+    type: ClusterIP
+    annotations: {}
+    ports:
+      - port: 80
+        targetPort: http
+        protocol: TCP
+        name: http
+        #nodePort: someFixedPortForUseWithTerraformCdkCfnEtc
+  ingress:
+    enabled: false
+    ingressClassName: ""
+    annotations: {}
+      # kubernetes.io/ingress.class: nginx
+      # kubernetes.io/tls-acme: "true"
+    hosts:
+      - host: chart-example.local
+        paths: []
+        #  - path: /*
+        #    pathType: ImplementationSpecific
+        # Extra paths that are not automatically connected to the server. This is useful when working with annotation based services.
+        extraPaths: []
+        # - path: /*
+        #   backend:
+        #     serviceName: ssl-redirect
+        #     servicePort: use-annotation
+        ## for Kubernetes >=1.19 (when "networking.k8s.io/v1" is used)
+        # - path: /*
+        #   pathType: Prefix
+        #   backend:
+        #     service:
+        #       name: ssl-redirect
+        #       port:
+        #         name: use-annotation
+    tls: []
+    #  - secretName: chart-example-tls
+    #    hosts:
+    #      - chart-example.local
+

cmd/actionsmetricsserver/main.go

@@ -0,0 +1,226 @@
+/*
+Copyright 2022 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"sync"
+
+	actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/actions/actions-runner-controller/github"
+	"github.com/actions/actions-runner-controller/logging"
+	"github.com/actions/actions-runner-controller/pkg/actionsmetrics"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+
+	"github.com/kelseyhightower/envconfig"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/exec"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
+	ctrl "sigs.k8s.io/controller-runtime"
+	// +kubebuilder:scaffold:imports
+)
+
+var (
+	scheme = runtime.NewScheme()
+)
+
+const (
+	webhookSecretTokenEnvName = "GITHUB_WEBHOOK_SECRET_TOKEN"
+)
+
+func init() {
+	_ = clientgoscheme.AddToScheme(scheme)
+
+	_ = actionsv1alpha1.AddToScheme(scheme)
+	// +kubebuilder:scaffold:scheme
+}
+
+func main() {
+	var (
+		err error
+
+		webhookAddr string
+		metricsAddr string
+
+		// The secret token of the GitHub Webhook. See https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks
+		webhookSecretToken    string
+		webhookSecretTokenEnv string
+
+		logLevel  string
+		logFormat string
+
+		ghClient *github.Client
+	)
+
+	var c github.Config
+	err = envconfig.Process("github", &c)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: processing environment variables: %v\n", err)
+		os.Exit(1)
+	}
+
+	webhookSecretTokenEnv = os.Getenv(webhookSecretTokenEnvName)
+
+	flag.StringVar(&webhookAddr, "webhook-addr", ":8000", "The address the metric endpoint binds to.")
+	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&logLevel, "log-level", logging.LogLevelDebug, `The verbosity of the logging. Valid values are "debug", "info", "warn", "error". Defaults to "debug".`)
+	flag.StringVar(&webhookSecretToken, "github-webhook-secret-token", "", "The personal access token of GitHub.")
+	flag.StringVar(&c.Token, "github-token", c.Token, "The personal access token of GitHub.")
+	flag.Int64Var(&c.AppID, "github-app-id", c.AppID, "The application ID of GitHub App.")
+	flag.Int64Var(&c.AppInstallationID, "github-app-installation-id", c.AppInstallationID, "The installation ID of GitHub App.")
+	flag.StringVar(&c.AppPrivateKey, "github-app-private-key", c.AppPrivateKey, "The path of a private key file to authenticate as a GitHub App")
+	flag.StringVar(&c.URL, "github-url", c.URL, "GitHub URL to be used for GitHub API calls")
+	flag.StringVar(&c.UploadURL, "github-upload-url", c.UploadURL, "GitHub Upload URL to be used for GitHub API calls")
+	flag.StringVar(&c.BasicauthUsername, "github-basicauth-username", c.BasicauthUsername, "Username for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API")
+	flag.StringVar(&c.BasicauthPassword, "github-basicauth-password", c.BasicauthPassword, "Password for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API")
+	flag.StringVar(&c.RunnerGitHubURL, "runner-github-url", c.RunnerGitHubURL, "GitHub URL to be used by runners during registration")
+	flag.StringVar(&logFormat, "log-format", "text", `The log format. Valid options are "text" and "json". Defaults to "text"`)
+
+	flag.Parse()
+
+	logger, err := logging.NewLogger(logLevel, logFormat)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: creating logger: %v\n", err)
+		os.Exit(1)
+	}
+	logger.WithName("setup")
+
+	if webhookSecretToken == "" && webhookSecretTokenEnv != "" {
+		logger.Info(fmt.Sprintf("Using the value from %s for -github-webhook-secret-token", webhookSecretTokenEnvName))
+		webhookSecretToken = webhookSecretTokenEnv
+	}
+
+	if webhookSecretToken == "" {
+		logger.Info(fmt.Sprintf("-github-webhook-secret-token and %s are missing or empty. Create one following https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks and specify it via the flag or the envvar", webhookSecretTokenEnvName))
+	}
+
+	ctrl.SetLogger(logger)
+
+	// Valid GitHub API credentials is required to call get workflow job logs
+	if len(c.Token) > 0 || (c.AppID > 0 && c.AppInstallationID > 0 && c.AppPrivateKey != "") || (len(c.BasicauthUsername) > 0 && len(c.BasicauthPassword) > 0) {
+		c.Log = &logger
+
+		ghClient, err = c.NewClient()
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "Error: Client creation failed.", err)
+			logger.Error(err, "unable to create controller", "controller", "Runner")
+			os.Exit(1)
+		}
+	} else {
+		logger.Info("GitHub client is not initialized. Runner groups with custom visibility are not supported. If needed, please provide GitHub authentication. This will incur in extra GitHub API calls")
+	}
+
+	eventReader := &actionsmetrics.EventReader{
+		Log:          ctrl.Log.WithName("workflowjobmetrics-eventreader"),
+		GitHubClient: ghClient,
+		Events:       make(chan interface{}, 1024*1024),
+	}
+
+	webhookServer := &actionsmetrics.WebhookServer{
+		Log:            ctrl.Log.WithName("workflowjobmetrics-webhookserver"),
+		SecretKeyBytes: []byte(webhookSecretToken),
+		GitHubClient:   ghClient,
+		EventHooks:     []actionsmetrics.EventHook{eventReader.HandleWorkflowJobEvent},
+	}
+
+	var wg sync.WaitGroup
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+		eventReader.ProcessWorkflowJobEvents(ctx)
+	}()
+
+	// Metrics Server
+
+	metricsHandler := promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{
+		ErrorHandling: promhttp.HTTPErrorOnError,
+	})
+
+	metricsMux := http.NewServeMux()
+	metricsMux.HandleFunc("/", metricsHandler.ServeHTTP)
+
+	metricsSrv := http.Server{
+		Addr:    metricsAddr,
+		Handler: metricsMux,
+	}
+
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+
+		go func() {
+			<-ctx.Done()
+
+			metricsSrv.Shutdown(context.Background())
+		}()
+
+		if err := metricsSrv.ListenAndServe(); err != nil {
+			if !errors.Is(err, http.ErrServerClosed) {
+				logger.Error(err, "problem running metrics server")
+			}
+		}
+	}()
+
+	// Webhook Server
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", webhookServer.Handle)
+
+	srv := http.Server{
+		Addr:    webhookAddr,
+		Handler: mux,
+	}
+
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+
+		go func() {
+			<-ctx.Done()
+
+			srv.Shutdown(context.Background())
+		}()
+
+		if err := srv.ListenAndServe(); err != nil {
+			if !errors.Is(err, http.ErrServerClosed) {
+				logger.Error(err, "problem running http server")
+			}
+		}
+	}()
+
+	go func() {
+		<-ctrl.SetupSignalHandler().Done()
+		cancel()
+	}()
+
+	wg.Wait()
+}

cmd/githubwebhookserver/main.go

@@ -26,11 +26,13 @@ import (
 	"sync"
 	"time"
 
-	actionsv1alpha1 "github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/actions-runner-controller/actions-runner-controller/controllers"
-	"github.com/actions-runner-controller/actions-runner-controller/github"
-	"github.com/actions-runner-controller/actions-runner-controller/logging"
+	actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	actionssummerwindnet "github.com/actions/actions-runner-controller/controllers/actions.summerwind.net"
+	"github.com/actions/actions-runner-controller/github"
+	"github.com/actions/actions-runner-controller/logging"
+
 	"github.com/kelseyhightower/envconfig"
+
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/exec"
@@ -41,8 +43,7 @@ import (
 )
 
 var (
-	scheme   = runtime.NewScheme()
-	setupLog = ctrl.Log.WithName("setup")
+	scheme = runtime.NewScheme()
 )
 
 const (
@@ -69,9 +70,9 @@ func main() {
 
 		watchNamespace string
 
-		enableLeaderElection bool
-		syncPeriod           time.Duration
-		logLevel             string
+		logLevel   string
+		queueLimit int
+		logFormat  string
 
 		ghClient *github.Client
 	)
@@ -88,10 +89,8 @@ func main() {
 	flag.StringVar(&webhookAddr, "webhook-addr", ":8000", "The address the metric endpoint binds to.")
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&watchNamespace, "watch-namespace", "", "The namespace to watch for HorizontalRunnerAutoscaler's to scale on Webhook. Set to empty for letting it watch for all namespaces.")
-	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
-		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
-	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change")
 	flag.StringVar(&logLevel, "log-level", logging.LogLevelDebug, `The verbosity of the logging. Valid values are "debug", "info", "warn", "error". Defaults to "debug".`)
+	flag.IntVar(&queueLimit, "queue-limit", actionssummerwindnet.DefaultQueueLimit, `The maximum length of the scale operation queue. The scale opration is enqueued per every matching webhook event, and the server returns a 500 HTTP status when the queue was already full on enqueue attempt.`)
 	flag.StringVar(&webhookSecretToken, "github-webhook-secret-token", "", "The personal access token of GitHub.")
 	flag.StringVar(&c.Token, "github-token", c.Token, "The personal access token of GitHub.")
 	flag.Int64Var(&c.AppID, "github-app-id", c.AppID, "The application ID of GitHub App.")
@@ -102,26 +101,32 @@ func main() {
 	flag.StringVar(&c.BasicauthUsername, "github-basicauth-username", c.BasicauthUsername, "Username for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API")
 	flag.StringVar(&c.BasicauthPassword, "github-basicauth-password", c.BasicauthPassword, "Password for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API")
 	flag.StringVar(&c.RunnerGitHubURL, "runner-github-url", c.RunnerGitHubURL, "GitHub URL to be used by runners during registration")
+	flag.StringVar(&logFormat, "log-format", "text", `The log format. Valid options are "text" and "json". Defaults to "text"`)
 
 	flag.Parse()
 
+	logger, err := logging.NewLogger(logLevel, logFormat)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: creating logger: %v\n", err)
+		os.Exit(1)
+	}
+	logger.WithName("setup")
+
 	if webhookSecretToken == "" && webhookSecretTokenEnv != "" {
-		setupLog.Info(fmt.Sprintf("Using the value from %s for -github-webhook-secret-token", webhookSecretTokenEnvName))
+		logger.Info(fmt.Sprintf("Using the value from %s for -github-webhook-secret-token", webhookSecretTokenEnvName))
 		webhookSecretToken = webhookSecretTokenEnv
 	}
 
 	if webhookSecretToken == "" {
-		setupLog.Info(fmt.Sprintf("-github-webhook-secret-token and %s are missing or empty. Create one following https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks and specify it via the flag or the envvar", webhookSecretTokenEnvName))
+		logger.Info(fmt.Sprintf("-github-webhook-secret-token and %s are missing or empty. Create one following https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks and specify it via the flag or the envvar", webhookSecretTokenEnvName))
 	}
 
 	if watchNamespace == "" {
-		setupLog.Info("-watch-namespace is empty. HorizontalRunnerAutoscalers in all the namespaces are watched, cached, and considered as scale targets.")
+		logger.Info("-watch-namespace is empty. HorizontalRunnerAutoscalers in all the namespaces are watched, cached, and considered as scale targets.")
 	} else {
-		setupLog.Info("-watch-namespace is %q. Only HorizontalRunnerAutoscalers in %q are watched, cached, and considered as scale targets.")
+		logger.Info("-watch-namespace is %q. Only HorizontalRunnerAutoscalers in %q are watched, cached, and considered as scale targets.")
 	}
 
-	logger := logging.NewLogger(logLevel)
-
 	ctrl.SetLogger(logger)
 
 	// In order to support runner groups with custom visibility (selected repositories), we need to perform some GitHub API calls.
@@ -135,27 +140,27 @@ func main() {
 		ghClient, err = c.NewClient()
 		if err != nil {
 			fmt.Fprintln(os.Stderr, "Error: Client creation failed.", err)
-			setupLog.Error(err, "unable to create controller", "controller", "Runner")
+			logger.Error(err, "unable to create controller", "controller", "Runner")
 			os.Exit(1)
 		}
 	} else {
-		setupLog.Info("GitHub client is not initialized. Runner groups with custom visibility are not supported. If needed, please provide GitHub authentication. This will incur in extra GitHub API calls")
+		logger.Info("GitHub client is not initialized. Runner groups with custom visibility are not supported. If needed, please provide GitHub authentication. This will incur in extra GitHub API calls")
 	}
 
+	syncPeriod := 10 * time.Minute
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		SyncPeriod:         &syncPeriod,
-		LeaderElection:     enableLeaderElection,
 		Namespace:          watchNamespace,
 		MetricsBindAddress: metricsAddr,
 		Port:               9443,
 	})
 	if err != nil {
-		setupLog.Error(err, "unable to start manager")
+		logger.Error(err, "unable to start manager")
 		os.Exit(1)
 	}
 
-	hraGitHubWebhook := &controllers.HorizontalRunnerAutoscalerGitHubWebhook{
+	hraGitHubWebhook := &actionssummerwindnet.HorizontalRunnerAutoscalerGitHubWebhook{
 		Name:           "webhookbasedautoscaler",
 		Client:         mgr.GetClient(),
 		Log:            ctrl.Log.WithName("controllers").WithName("webhookbasedautoscaler"),
@@ -164,10 +169,11 @@ func main() {
 		SecretKeyBytes: []byte(webhookSecretToken),
 		Namespace:      watchNamespace,
 		GitHubClient:   ghClient,
+		QueueLimit:     queueLimit,
 	}
 
 	if err = hraGitHubWebhook.SetupWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "webhookbasedautoscaler")
+		logger.Error(err, "unable to create controller", "controller", "webhookbasedautoscaler")
 		os.Exit(1)
 	}
 
@@ -180,9 +186,9 @@ func main() {
 		defer cancel()
 		defer wg.Done()
 
-		setupLog.Info("starting webhook server")
+		logger.Info("starting webhook server")
 		if err := mgr.Start(ctx); err != nil {
-			setupLog.Error(err, "problem running manager")
+			logger.Error(err, "problem running manager")
 			os.Exit(1)
 		}
 	}()
@@ -208,7 +214,7 @@ func main() {
 
 		if err := srv.ListenAndServe(); err != nil {
 			if !errors.Is(err, http.ErrServerClosed) {
-				setupLog.Error(err, "problem running http server")
+				logger.Error(err, "problem running http server")
 			}
 		}
 	}()

config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -61,6 +61,16 @@ spec:
                         type: integer
                     type: object
                   type: array
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 maxReplicas:
                   description: MaxReplicas is the maximum number of replicas the deployment is allowed to scale
                   type: integer
@@ -92,7 +102,7 @@ spec:
                         description: ScaleUpThreshold is the percentage of busy runners greater than which will trigger the hpa to scale runners up.
                         type: string
                       type:
-                        description: Type is the type of metric to be used for autoscaling. The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+                        description: Type is the type of metric to be used for autoscaling. It can be TotalNumberOfQueuedAndInProgressWorkflowRuns or PercentageRunnersBusy.
                         type: string
                     type: object
                   type: array
@@ -170,7 +180,7 @@ spec:
                 scheduledOverrides:
                   description: ScheduledOverrides is the list of ScheduledOverride. It can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. The earlier a scheduled override is, the higher it is prioritized.
                   items:
-                    description: ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally be recurring, so that the correspoding override happens every day, week, month, or year.
+                    description: ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally be recurring, so that the corresponding override happens every day, week, month, or year.
                     properties:
                       endTime:
                         description: EndTime is the time at which the first override ends.

config/default/kustomization.yaml

@@ -22,8 +22,6 @@ bases:
 - ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
-# [GH_WEBHOOK_SERVER] To enable the GitHub webhook server, uncomment all sections with 'GH_WEBHOOK_SERVER'.
-#- ../github-webhook-server
 
 patchesStrategicMerge:
 # Protect the /metrics endpoint by putting it behind auth.
@@ -46,10 +44,6 @@ patchesStrategicMerge:
 # 'CERTMANAGER' needs to be enabled to use ca injection
 - webhookcainjection_patch.yaml
 
-# [GH_WEBHOOK_SERVER] To enable the GitHub webhook server, uncomment all sections with 'GH_WEBHOOK_SERVER'.
-# Protect the GitHub webhook server metrics endpoint by putting it behind auth.
-# - gh-webhook-server-auth-proxy-patch.yaml
-
 # the following config is for teaching kustomize how to do var substitution
 vars:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.

config/github-webhook-server/kustomization.yaml

@@ -2,11 +2,14 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 images:
-  - name: controller
-    newName: summerwind/actions-runner-controller
-    newTag: latest
+- name: controller
+  newName: summerwind/actions-runner-controller
+  newTag: latest
 
 resources:
-  - deployment.yaml
-  - rbac.yaml
-  - service.yaml
+- deployment.yaml
+- rbac.yaml
+- service.yaml
+
+patchesStrategicMerge:
+- gh-webhook-server-auth-proxy-patch.yaml

config/manager/kustomization.yaml

@@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: summerwind/actions-runner-controller
-  newTag: latest
+  newName: jokicnikola07/actions-runner-controller
+  newTag: dev

config/rbac/role.yaml

@@ -249,3 +249,36 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - create
+  - delete
+  - get

config/samples/actions_v1alpha1_runner.yaml

@@ -3,4 +3,4 @@ kind: Runner
 metadata:
   name: summerwind-actions-runner-controller
 spec:
-  repository: actions-runner-controller/actions-runner-controller
+  repository: actions/actions-runner-controller

config/samples/actions_v1alpha1_runnerdeployment.yaml

@@ -6,4 +6,4 @@ spec:
   replicas: 2
   template:
     spec:
-      repository: actions-runner-controller/actions-runner-controller
+      repository: actions/actions-runner-controller

config/samples/actions_v1alpha1_runnerreplicaset.yaml

@@ -6,4 +6,4 @@ spec:
   replicas: 2
   template:
     spec:
-      repository: actions-runner-controller/actions-runner-controller
+      repository: actions/actions-runner-controller

contrib/README.md

@@ -0,0 +1,6 @@
+The `contrib` directory is the place for sharing various example code for deploying and operating `actions-runner-controller`.
+
+Anything contained in this directory is provided as-is. The maintainers of `actions-runner-controller` is not yet commited to provide
+full support for using, fixing, and enhancing it. However, they will do their best effort to collect feedbacks from early adopters and advanced users like you, and may eventually consider graduating any of the examples as an official addition to the project.
+
+See https://github.com/actions/actions-runner-controller/pull/1375#issuecomment-1258816470 and https://github.com/actions/actions-runner-controller/pull/1559#issuecomment-1258827496 for more context.

contrib/examples/actions-runner/.helmignore

@@ -0,0 +1,25 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+# Docs
+docs/

contrib/examples/actions-runner/Chart.yaml

@@ -0,0 +1,10 @@
+apiVersion: v2
+name: actions-runner
+description: Helm Chart for Github Actions Runner
+type: application
+version: 0.0.1
+appVersion: 2.290.1
+
+home: https://github.com/actions/actions-runner-controller/tree/master/runner
+sources:
+  - https://github.com/actions/actions-runner-controller/tree/master/runner

contrib/examples/actions-runner/README.md

@@ -0,0 +1,36 @@
+## Docs
+
+All additional docs are kept in the `docs/` folder, this README is solely for documenting the values.yaml keys and values
+
+## Values
+
+**_The values are documented as of HEAD, to review the configuration options for your chart version ensure you view this file at the relevent [tag](https://github.com/actions/actions-runner-controller/tags)_**
+
+> _Default values are the defaults set in the charts values.yaml, some properties have default configurations in the code for when the property is omitted or invalid_
+
+| Key                                                      | Description                                                                                                                | Default                                                              |
+|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------|
+| `labels`                                                 | Set labels to apply to all resources in the chart                                                                          |                                                                      |
+| `replicaCount`                                           | Set the number of runner pods                                                                                          | 1                                                                    |
+| `image.repository`                                       | The "repository/image" of the runner container                                                                         | summerwind/actions-runner                                 |
+| `image.tag`                                              | The tag of the runner container                                                                                        |                                                                      |
+| `image.pullPolicy`                                       | The pull policy of the runner image                                                                                    | IfNotPresent                                                         |
+| `imagePullSecrets`                                       | Specifies the secret to be used when pulling the runner pod containers                                                 |                                                                      |
+| `fullnameOverride`                                       | Override the full resource names	                                                                                        |                                                                      |
+| `nameOverride`                                           | Override the resource name prefix	                                                                                        |                                                                      |
+| `podAnnotations`                                         | Set annotations for the runner pod                                                                                     |                                                                      |
+| `podLabels`                                              | Set labels for the runner pod                                                                                          |                                                                      |
+| `podSecurityContext`                                     | Set the security context to runner pod                                                                                 |                                                                      |
+| `nodeSelector`                                           | Set the  pod nodeSelector                                                                                        |                                                                      |
+| `affinity`                                               | Set the runner pod affinity rules                                                                                      |                                                                      |
+| `tolerations`                                            | Set the runner pod tolerations                                                                                         |                                                                      |
+| `env`                                                    | Set environment variables for the runner container                                                                     |                                                                      |
+| `organization`                                           | Github organization where runner will be registered                                                                        | test                                                       |
+| `repository`                                             | Github repository where runner will be registered                                                                        |                                                          |
+| `runnerLabels`                                           | Labels you want to add in your runner                                                                       | test                                                       |
+| `autoscaler.enabled`                                     | Enable the HorizontalRunnerAutoscaler, if its enabled then replica count will not be used                                                                    | true                                                       |
+| `autoscaler.minReplicas`                                 | Minimum no of replicas                                                                    | 1                                                      |
+| `autoscaler.maxReplicas`                                 | Maximum no of replicas                                                                    | 5                                                      |
+| `autoscaler.scaleDownDelaySecondsAfterScaleOut`          | [Anti-Flapping Configuration](https://github.com/actions/actions-runner-controller#anti-flapping-configuration)                                                                   | 120                                                     |
+| `autoscaler.metrics`                                 | [Pull driven scaling](https://github.com/actions/actions-runner-controller#pull-driven-scaling)                                                                    | default                                                      |
+| `autoscaler.scaleUpTriggers`                         | [Webhook driven scaling](https://github.com/actions/actions-runner-controller#webhook-driven-scaling)                                                                    |                                                     |

contrib/examples/actions-runner/templates/_helpers.tpl

@@ -0,0 +1,54 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "actions-runner.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "actions-runner.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "actions-runner.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "actions-runner.labels" -}}
+helm.sh/chart: {{ include "actions-runner.chart" . }}
+{{ include "actions-runner.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- range $k, $v := .Values.labels }}
+{{ $k }}: {{ $v }}
+{{- end }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "actions-runner.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "actions-runner.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

contrib/examples/actions-runner/templates/deployment.yaml

@@ -0,0 +1,96 @@
+apiVersion: v1
+kind: List
+items:
+- apiVersion: actions.summerwind.dev/v1alpha1
+  kind: RunnerDeployment
+  metadata:
+    name: {{ include "actions-runner.fullname" . }}
+    namespace: {{ .Release.Namespace }}
+    labels:
+      {{- include "actions-runner.labels" . | nindent 6 }}
+  spec:
+    {{- if not .Values.autoscaler.enabled }}
+    replicas: {{ .Values.replicaCount }}
+    {{- end }}
+    selector:
+      matchLabels:
+        {{- include "actions-runner.selectorLabels" . | nindent 8 }}
+    template:
+      metadata:
+        {{- with .Values.podAnnotations }}
+        annotations:
+          kubectl.kubernetes.io/default-logs-container: "runner"
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        labels:
+          {{- include "actions-runner.selectorLabels" . | nindent 10 }}
+        {{- with .Values.podLabels }}
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+      spec:
+        {{- with .Values.imagePullSecrets }}
+        imagePullSecrets:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        securityContext:
+          {{- toYaml .Values.podSecurityContext | nindent 10 }}
+        {{- with .Values.priorityClassName }}
+        priorityClassName: "{{ . }}"
+        {{- end }}
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (cat "v" .Chart.AppVersion | replace " " "") }}"
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        {{- if .Values.organization }}
+        organization: {{ .Values.organization }}
+        {{- end }}
+        {{- if .Values.repository }}
+        repository: {{ .Values.repository }}
+        {{- end }}
+        group: {{ .Values.group | default "Default" }}
+        {{- with .Values.runnerLabels }}
+        labels:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.nodeSelector }}
+        nodeSelector:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.affinity }}
+        affinity:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.tolerations }}
+        tolerations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- if .Values.env }}
+        env:
+        {{- range $key, $val := .Values.env }}
+        - name: {{ $key }}
+          value: {{ $val | quote }}
+        {{- end }}
+        {{- end }}
+{{- if .Values.autoscaler.enabled }}
+- apiVersion: actions.summerwind.dev/v1alpha1
+  kind: HorizontalRunnerAutoscaler
+  metadata:
+    name: {{ (list (include "actions-runner.fullname" .) "autoscaler" | join "-") }}
+    namespace: {{ .Release.Namespace }}
+    labels:
+      {{- include "actions-runner.labels" . | nindent 6 }}
+  spec:
+    {{- if .Values.autoscaler.scaleDownDelaySecondsAfterScaleOut }}
+    scaleDownDelaySecondsAfterScaleOut: {{ .Values.autoscaler.scaleDownDelaySecondsAfterScaleOut }}
+    {{- end }}
+    scaleTargetRef:
+      name: {{ include "actions-runner.fullname" . }}
+    minReplicas: {{ .Values.autoscaler.minReplicas }}
+    maxReplicas: {{ .Values.autoscaler.maxReplicas }}
+    {{- with .Values.autoscaler.scaleUpTriggers }}
+    scaleUpTriggers:
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+    {{- with .Values.autoscaler.metrics }}
+    metrics:
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+{{- end }}

contrib/examples/actions-runner/values.yaml

@@ -0,0 +1,63 @@
+image:
+  repository: summerwind/actions-runner
+  tag: v2.290.1-ubuntu-20.04
+  pullPolicy: IfNotPresent
+
+# Create runner for an organization or a repository
+# Set only one of the two either organization or repository
+# By default, it creates runner under github organization test
+organization: test
+# repository: mumoshu/actions-runner-controller-ci
+
+# Labels you want to add in your runner
+runnerLabels:
+  - test
+
+# If you enable Autoscaler, then it will not be used
+replicaCount: 1
+
+# The Runner Group that the runner(s) should be associated with.
+# See https://docs.github.com/en/github-ae@latest/actions/hosting-your-own-runners/managing-access-to-self-hosted-runners-using-groups.
+group: Default
+
+autoscaler:
+  enabled: true
+  minReplicas: 1
+  maxReplicas: 5
+  scaleDownDelaySecondsAfterScaleOut: 120
+  # metrics (pull method) / scaleUpTriggers (push method)
+  # https://github.com/actions/actions-runner-controller#pull-driven-scaling
+  # https://github.com/actions/actions-runner-controller#webhook-driven-scaling
+  metrics:
+  - type: PercentageRunnersBusy
+    scaleUpThreshold: '0.75'
+    scaleDownThreshold: '0.25'
+    scaleUpFactor: '2'
+    scaleDownFactor: '0.5'
+  # scaleUpTriggers:
+  # - githubEvent: {}
+  #   duration: "5m"
+
+podAnnotations: {}
+
+podLabels: {}
+
+imagePullSecrets: []
+
+podSecurityContext:
+  {}
+  # fsGroup: 2000
+
+# Leverage a PriorityClass to ensure your pods survive resource shortages
+# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+# PriorityClass: system-cluster-critical
+priorityClassName: ""
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+env:
+  {}

contrib/examples/terraform/actions-runner-controller.tf

@@ -0,0 +1,60 @@
+### Deploying with exposed github token
+
+resource "kubernetes_namespace" "arc" {
+ metadata {
+   name = "actions-runner-system"
+ }
+}
+
+resource "helm_release" "actions-runner-controller" {
+ count            = var.actions_runner_controller
+ name             = "actions-runner-controller"
+ namespace        = kubernetes_namespace.arc.metadata[0].name
+ create_namespace = true
+ chart            = "actions-runner-controller"
+ repository       = "https://actions-runner-controller.github.io/actions-runner-controller"
+ version          = "v0.19.1"
+ values = [<<EOF
+   authSecret:
+     github_token: hdjasyd7das7d7asd78as87dasdas
+     create: true
+ EOF
+ ]
+ depends_on = [resource.helm_release.cm]
+}
+
+#============================================================================================================================================
+### Deploying with secret manager like AWS's
+# make sure the name of the secret is the same as secret_id
+
+data "aws_secretsmanager_secret_version" "creds" {
+  secret_id = "github/access_token"
+}
+locals {
+  github_creds = jsondecode(
+    data.aws_secretsmanager_secret_version.creds.secret_string
+  )
+}
+
+resource "kubernetes_namespace" "arc" {
+  metadata {
+    name = "actions-runner-system"
+  }
+}
+
+resource "helm_release" "actions-runner-controller" {
+  count            = var.actions_runner_controller
+  name             = "actions-runner-controller"
+  namespace        = kubernetes_namespace.arc.metadata[0].name
+  create_namespace = true
+  chart            = "actions-runner-controller"
+  repository       = "https://actions-runner-controller.github.io/actions-runner-controller"
+  version          = "v0.19.1"
+  values = [<<EOF
+    authSecret:
+      github_token: ${local.github_creds.github_token}
+      create: true
+  EOF
+  ]
+  depends_on = [resource.helm_release.cm]
+}

contrib/examples/terraform/cert-manager.tf

@@ -0,0 +1,27 @@
+# cert-manager must be deployed or included via the deployment process
+
+resource "kubernetes_namespace" "cm" {
+  metadata {
+    name = "cert-manager"
+  }
+}
+
+resource "helm_release" "cm" {
+  count            = var.actions_runner_controller
+  name             = "cm"
+  namespace        = kubernetes_namespace.cm.metadata[0].name
+  create_namespace = true
+  chart            = "cert-manager"
+  repository       = "https://charts.jetstack.io"
+  version          = "v1.8.0"
+  values = [<<EOF
+    global:
+      podSecurityPolicy:
+        enabled: true
+        useAppArmor: true
+    prometheus:
+      enabled: false
+    installCRDs: true
+  EOF
+  ]
+}

controllers/actions.summerwind.net/autoscaling.go

@@ -1,4 +1,4 @@
-package controllers
+package actionssummerwindnet
 
 import (
 	"context"
@@ -8,8 +8,12 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/google/go-github/v39/github"
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	prometheus_metrics "github.com/actions/actions-runner-controller/controllers/actions.summerwind.net/metrics"
+	arcgithub "github.com/actions/actions-runner-controller/github"
+	"github.com/google/go-github/v47/github"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -19,7 +23,7 @@ const (
 	defaultScaleDownFactor    = 0.7
 )
 
-func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
 	if hra.Spec.MinReplicas == nil {
 		return nil, fmt.Errorf("horizontalrunnerautoscaler %s/%s is missing minReplicas", hra.Namespace, hra.Name)
 	} else if hra.Spec.MaxReplicas == nil {
@@ -30,7 +34,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTa
 	numMetrics := len(metrics)
 	if numMetrics == 0 {
 		// We don't default to anything since ARC 0.23.0
-		// See https://github.com/actions-runner-controller/actions-runner-controller/issues/728
+		// See https://github.com/actions/actions-runner-controller/issues/728
 		return nil, nil
 	} else if numMetrics > 2 {
 		return nil, fmt.Errorf("too many autoscaling metrics configured: It must be 0 to 2, but got %d", numMetrics)
@@ -46,9 +50,9 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTa
 
 	switch primaryMetricType {
 	case v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns:
-		suggested, err = r.suggestReplicasByQueuedAndInProgressWorkflowRuns(st, hra, &primaryMetric)
+		suggested, err = r.suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc, st, hra, &primaryMetric)
 	case v1alpha1.AutoscalingMetricTypePercentageRunnersBusy:
-		suggested, err = r.suggestReplicasByPercentageRunnersBusy(st, hra, primaryMetric)
+		suggested, err = r.suggestReplicasByPercentageRunnersBusy(ghc, st, hra, primaryMetric)
 	default:
 		return nil, fmt.Errorf("validating autoscaling metrics: unsupported metric type %q", primaryMetric)
 	}
@@ -81,11 +85,10 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTa
 		)
 	}
 
-	return r.suggestReplicasByQueuedAndInProgressWorkflowRuns(st, hra, &fallbackMetric)
+	return r.suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc, st, hra, &fallbackMetric)
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgressWorkflowRuns(st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics *v1alpha1.MetricSpec) (*int, error) {
-
+func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics *v1alpha1.MetricSpec) (*int, error) {
 	var repos [][]string
 	repoID := st.repo
 	if repoID == "" {
@@ -96,7 +99,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 
 		// In case it's an organizational runners deployment without any scaling metrics defined,
 		// we assume that the desired replicas should always be `minReplicas + capacityReservedThroughWebhook`.
-		// See https://github.com/actions-runner-controller/actions-runner-controller/issues/377#issuecomment-793372693
+		// See https://github.com/actions/actions-runner-controller/issues/377#issuecomment-793372693
 		if metrics == nil {
 			return nil, nil
 		}
@@ -124,7 +127,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 		opt := github.ListWorkflowJobsOptions{ListOptions: github.ListOptions{PerPage: 50}}
 		var allJobs []*github.WorkflowJob
 		for {
-			jobs, resp, err := r.GitHubClient.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, &opt)
+			jobs, resp, err := ghc.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, &opt)
 			if err != nil {
 				r.Log.Error(err, "Error listing workflow jobs")
 				return //err
@@ -182,7 +185,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 
 	for _, repo := range repos {
 		user, repoName := repo[0], repo[1]
-		workflowRuns, err := r.GitHubClient.ListRepositoryWorkflowRuns(context.TODO(), user, repoName)
+		workflowRuns, err := ghc.ListRepositoryWorkflowRuns(context.TODO(), user, repoName)
 		if err != nil {
 			return nil, err
 		}
@@ -209,6 +212,20 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 
 	necessaryReplicas := queued + inProgress
 
+	prometheus_metrics.SetHorizontalRunnerAutoscalerQueuedAndInProgressWorkflowRuns(
+		hra.ObjectMeta,
+		st.enterprise,
+		st.org,
+		st.repo,
+		st.kind,
+		st.st,
+		necessaryReplicas,
+		completed,
+		inProgress,
+		queued,
+		unknown,
+	)
+
 	r.Log.V(1).Info(
 		fmt.Sprintf("Suggested desired replicas of %d by TotalNumberOfQueuedAndInProgressWorkflowRuns", necessaryReplicas),
 		"workflow_runs_completed", completed,
@@ -224,7 +241,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 	return &necessaryReplicas, nil
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunnersBusy(st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics v1alpha1.MetricSpec) (*int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunnersBusy(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics v1alpha1.MetricSpec) (*int, error) {
 	ctx := context.Background()
 	scaleUpThreshold := defaultScaleUpThreshold
 	scaleDownThreshold := defaultScaleDownThreshold
@@ -293,7 +310,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 	)
 
 	// ListRunners will return all runners managed by GitHub - not restricted to ns
-	runners, err := r.GitHubClient.ListRunners(
+	runners, err := ghc.ListRunners(
 		ctx,
 		enterprise,
 		organization,
@@ -314,22 +331,52 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 		numRunners           int
 		numRunnersRegistered int
 		numRunnersBusy       int
+		numTerminatingBusy   int
 	)
 
 	numRunners = len(runnerMap)
 
+	busyTerminatingRunnerPods := map[string]struct{}{}
+
+	kindLabel := LabelKeyRunnerDeploymentName
+	if hra.Spec.ScaleTargetRef.Kind == "RunnerSet" {
+		kindLabel = LabelKeyRunnerSetName
+	}
+
+	var runnerPodList corev1.PodList
+	if err := r.Client.List(ctx, &runnerPodList, client.InNamespace(hra.Namespace), client.MatchingLabels(map[string]string{
+		kindLabel: hra.Spec.ScaleTargetRef.Name,
+	})); err != nil {
+		return nil, err
+	}
+
+	for _, p := range runnerPodList.Items {
+		if p.Annotations[AnnotationKeyUnregistrationFailureMessage] != "" {
+			busyTerminatingRunnerPods[p.Name] = struct{}{}
+		}
+	}
+
 	for _, runner := range runners {
 		if _, ok := runnerMap[*runner.Name]; ok {
 			numRunnersRegistered++
 
 			if runner.GetBusy() {
 				numRunnersBusy++
+			} else if _, ok := busyTerminatingRunnerPods[*runner.Name]; ok {
+				numTerminatingBusy++
 			}
+
+			delete(busyTerminatingRunnerPods, *runner.Name)
 		}
 	}
 
+	// Remaining busyTerminatingRunnerPods are runners that were not on the ListRunners API response yet
+	for range busyTerminatingRunnerPods {
+		numTerminatingBusy++
+	}
+
 	var desiredReplicas int
-	fractionBusy := float64(numRunnersBusy) / float64(desiredReplicasBefore)
+	fractionBusy := float64(numRunnersBusy+numTerminatingBusy) / float64(desiredReplicasBefore)
 	if fractionBusy >= scaleUpThreshold {
 		if scaleUpAdjustment > 0 {
 			desiredReplicas = desiredReplicasBefore + scaleUpAdjustment
@@ -350,6 +397,19 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 	//
 	// - num_runners can be as twice as large as replicas_desired_before while
 	//   the runnerdeployment controller is replacing RunnerReplicaSet for runner update.
+	prometheus_metrics.SetHorizontalRunnerAutoscalerPercentageRunnersBusy(
+		hra.ObjectMeta,
+		st.enterprise,
+		st.org,
+		st.repo,
+		st.kind,
+		st.st,
+		desiredReplicas,
+		numRunners,
+		numRunnersRegistered,
+		numRunnersBusy,
+		numTerminatingBusy,
+	)
 
 	r.Log.V(1).Info(
 		fmt.Sprintf("Suggested desired replicas of %d by PercentageRunnersBusy", desiredReplicas),
@@ -358,6 +418,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 		"num_runners", numRunners,
 		"num_runners_registered", numRunnersRegistered,
 		"num_runners_busy", numRunnersBusy,
+		"num_terminating_busy", numTerminatingBusy,
 		"namespace", hra.Namespace,
 		"kind", st.kind,
 		"name", st.st,

controllers/actions.summerwind.net/autoscaling_test.go

@@ -1,4 +1,4 @@
-package controllers
+package actionssummerwindnet
 
 import (
 	"context"
@@ -7,9 +7,9 @@ import (
 	"net/url"
 	"testing"
 
-	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/actions-runner-controller/actions-runner-controller/github"
-	"github.com/actions-runner-controller/actions-runner-controller/github/fake"
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/actions/actions-runner-controller/github"
+	"github.com/actions/actions-runner-controller/github/fake"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -330,7 +330,6 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 
 			h := &HorizontalRunnerAutoscalerReconciler{
 				Log:                   log,
-				GitHubClient:          client,
 				Scheme:                scheme,
 				DefaultScaleDownDelay: DefaultScaleDownDelay,
 			}
@@ -379,7 +378,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 
 			st := h.scaleTargetFromRD(context.Background(), rd)
 
-			got, err := h.computeReplicasWithCache(log, metav1Now.Time, st, hra, minReplicas)
+			got, err := h.computeReplicasWithCache(client, log, metav1Now.Time, st, hra, minReplicas)
 			if err != nil {
 				if tc.err == "" {
 					t.Fatalf("unexpected error: expected none, got %v", err)
@@ -720,7 +719,6 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			h := &HorizontalRunnerAutoscalerReconciler{
 				Log:                   log,
 				Scheme:                scheme,
-				GitHubClient:          client,
 				DefaultScaleDownDelay: DefaultScaleDownDelay,
 			}
 
@@ -781,7 +779,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 
 			st := h.scaleTargetFromRD(context.Background(), rd)
 
-			got, err := h.computeReplicasWithCache(log, metav1Now.Time, st, hra, minReplicas)
+			got, err := h.computeReplicasWithCache(client, log, metav1Now.Time, st, hra, minReplicas)
 			if err != nil {
 				if tc.err == "" {
 					t.Fatalf("unexpected error: expected none, got %v", err)

controllers/actions.summerwind.net/constants.go

@@ -1,20 +1,25 @@
-package controllers
+package actionssummerwindnet
 
 import "time"
 
 const (
 	LabelKeyRunnerSetName = "runnerset-name"
+	LabelKeyRunner        = "actions-runner"
 )
 
 const (
 	// This names requires at least one slash to work.
 	// See https://github.com/google/knative-gcp/issues/378
-	runnerPodFinalizerName = "actions.summerwind.dev/runner-pod"
+	runnerPodFinalizerName             = "actions.summerwind.dev/runner-pod"
+	runnerLinkedResourcesFinalizerName = "actions.summerwind.dev/linked-resources"
 
 	annotationKeyPrefix = "actions-runner/"
 
 	AnnotationKeyLastRegistrationCheckTime = "actions-runner-controller/last-registration-check-time"
 
+	// AnnotationKeyUnregistrationFailureMessage is the annotation that is added onto the pod once it failed to be unregistered from GitHub due to e.g. 422 error
+	AnnotationKeyUnregistrationFailureMessage = annotationKeyPrefix + "unregistration-failure-message"
+
 	// AnnotationKeyUnregistrationCompleteTimestamp is the annotation that is added onto the pod once the previously started unregistration process has been completed.
 	AnnotationKeyUnregistrationCompleteTimestamp = annotationKeyPrefix + "unregistration-complete-timestamp"
 
@@ -56,9 +61,12 @@ const (
 	// In case it actually took more than DefaultRunnerPodRecreationDelayAfterWebhookScale for the workflow_job completion event to arrive,
 	// ARC will recreate the completed runner(s), assuming something went wrong in either GitHub, your K8s cluster, or ARC, so ARC needs to resync anyway.
 	//
-	// See https://github.com/actions-runner-controller/actions-runner-controller/pull/1180
+	// See https://github.com/actions/actions-runner-controller/pull/1180
 	DefaultRunnerPodRecreationDelayAfterWebhookScale = 10 * time.Minute
 
 	EnvVarRunnerName  = "RUNNER_NAME"
 	EnvVarRunnerToken = "RUNNER_TOKEN"
+
+	// defaultHookPath is path to the hook script used when the "containerMode: kubernetes" is specified
+	defaultRunnerHookPath = "/runner/k8s/index.js"
 )