Add multi-label support to scalesets (#4408)

This commit is contained in:
Nikola Jokic
2026-03-19 15:29:40 +01:00
committed by GitHub
parent 9bc1c9e53e
commit 802dc28d38
28 changed files with 388 additions and 44 deletions

View File

@@ -66,6 +66,9 @@ type AutoscalingRunnerSetSpec struct {
// +optional
RunnerScaleSetName string `json:"runnerScaleSetName,omitempty"`
// +optional
RunnerScaleSetLabels []string `json:"runnerScaleSetLabels,omitempty"`
// +optional
Proxy *ProxyConfig `json:"proxy,omitempty"`

View File

@@ -227,6 +227,11 @@ func (in *AutoscalingRunnerSetList) DeepCopyObject() runtime.Object {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *AutoscalingRunnerSetSpec) DeepCopyInto(out *AutoscalingRunnerSetSpec) {
*out = *in
if in.RunnerScaleSetLabels != nil {
in, out := &in.RunnerScaleSetLabels, &out.RunnerScaleSetLabels
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.Proxy != nil {
in, out := &in.Proxy, &out.Proxy
*out = new(ProxyConfig)

View File

@@ -8385,6 +8385,10 @@ spec:
type: object
runnerGroup:
type: string
runnerScaleSetLabels:
items:
type: string
type: array
runnerScaleSetName:
type: string
template:

View File

@@ -8385,6 +8385,10 @@ spec:
type: object
runnerGroup:
type: string
runnerScaleSetLabels:
items:
type: string
type: array
runnerScaleSetName:
type: string
template:

View File

@@ -83,6 +83,18 @@ spec:
githubConfigSecret: {{ include "github-secret.name" . | quote }}
runnerGroup: {{ .Values.scaleset.runnerGroup | quote }}
runnerScaleSetName: {{ .Values.scaleset.name | quote }}
{{- if and .Values.scaleset.labels (kindIs "slice" .Values.scaleset.labels) }}
{{- range .Values.scaleset.labels }}
{{- if empty . }}
{{- fail "scaleset.labels contains an empty string, each label must be a non-empty string of less than 256 characters" }}
{{- end }}
{{- if ge (len .) 256 }}
{{- fail "scaleset.labels contains a label that is 256 characters or more, each label must be a non-empty string of less than 256 characters" }}
{{- end }}
{{- end }}
runnerScaleSetLabels:
{{- toYaml .Values.scaleset.labels | nindent 4 }}
{{- end }}
{{- if .Values.githubServerTLS }}
githubServerTLS:

View File

@@ -0,0 +1,58 @@
suite: "AutoscalingRunnerSet scale set labels"
templates:
- autoscalingrunnserset.yaml
tests:
- it: should apply scaleset labels slice as runnerScaleSetLabels
set:
scaleset.name: "test"
scaleset.labels:
- "linux"
- "x64"
auth.url: "https://github.com/org"
auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system"
release:
name: "test-name"
namespace: "test-namespace"
asserts:
- equal:
path: spec.runnerScaleSetLabels[0]
value: "linux"
- equal:
path: spec.runnerScaleSetLabels[1]
value: "x64"
- it: should fail when a scaleset label is empty
set:
scaleset.name: "test"
scaleset.labels:
- "linux"
- ""
auth.url: "https://github.com/org"
auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system"
release:
name: "test-name"
namespace: "test-namespace"
asserts:
- failedTemplate:
errorMessage: "scaleset.labels contains an empty string, each label must be a non-empty string of less than 256 characters"
- it: should fail when a scaleset label is 256 characters or more
set:
scaleset.name: "test"
scaleset.labels:
- "linux"
- "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
auth.url: "https://github.com/org"
auth.githubToken: "gh_token12345"
controllerServiceAccount.name: "arc"
controllerServiceAccount.namespace: "arc-system"
release:
name: "test-name"
namespace: "test-namespace"
asserts:
- failedTemplate:
errorMessage: "scaleset.labels contains a label that is 256 characters or more, each label must be a non-empty string of less than 256 characters"

View File

@@ -5,6 +5,10 @@ scaleset:
# Name of the scaleset
name: ""
runnerGroup: "default"
# Labels are optional list of strings that will be applied to the scaleset
# allowing scaleset to be selected by the listener based on the labels specified in the workflow.
# https://docs.github.com/en/actions/how-tos/manage-runners/self-hosted-runners/apply-labels
labels: []
## minRunners is the min number of idle runners. The target number of runners created will be
## calculated as a sum of minRunners and the number of jobs assigned to the scale set.
# minRunners: 0

View File

@@ -75,6 +75,18 @@ spec:
{{- with .Values.runnerScaleSetName }}
runnerScaleSetName: {{ . }}
{{- end }}
{{- if and .Values.scaleSetLabels (kindIs "slice" .Values.scaleSetLabels) }}
{{- range .Values.scaleSetLabels }}
{{- if empty . }}
{{- fail "scaleSetLabels contains an empty string, each label must be a non-empty string of less than 256 characters" }}
{{- end }}
{{- if ge (len .) 256 }}
{{- fail "scaleSetLabels contains a label that is 256 characters or more, each label must be a non-empty string of less than 256 characters" }}
{{- end }}
{{- end }}
runnerScaleSetLabels:
{{- toYaml .Values.scaleSetLabels | nindent 4 }}
{{- end }}
{{- if .Values.githubServerTLS }}
githubServerTLS:

View File

@@ -474,6 +474,37 @@ func TestTemplateRenderedAutoScalingRunnerSet_RunnerScaleSetName(t *testing.T) {
assert.Equal(t, "ghcr.io/actions/actions-runner:latest", ars.Spec.Template.Spec.Containers[0].Image)
}
func TestTemplateRenderedAutoScalingRunnerSet_ScaleSetLabels(t *testing.T) {
t.Parallel()
// Path to the helm chart we will test
helmChartPath, err := filepath.Abs("../../gha-runner-scale-set")
require.NoError(t, err)
releaseName := "test-runners"
namespaceName := "test-" + strings.ToLower(random.UniqueId())
options := &helm.Options{
Logger: logger.Discard,
SetValues: map[string]string{
"githubConfigUrl": "https://github.com/actions",
"githubConfigSecret.github_token": "gh_token12345",
"scaleSetLabels[0]": "linux",
"scaleSetLabels[1]": "x64",
"controllerServiceAccount.name": "arc",
"controllerServiceAccount.namespace": "arc-system",
},
KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
}
output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/autoscalingrunnerset.yaml"})
var ars v1alpha1.AutoscalingRunnerSet
helm.UnmarshalK8SYaml(t, output, &ars)
assert.Equal(t, []string{"linux", "x64"}, ars.Spec.RunnerScaleSetLabels)
}
func TestTemplateRenderedAutoScalingRunnerSet_ProvideMetadata(t *testing.T) {
t.Parallel()

View File

@@ -2,6 +2,8 @@
## ex: https://github.com/myorg/myrepo or https://github.com/myorg or https://github.com/enterprises/myenterprise
githubConfigUrl: ""
scaleSetLabels: []
## githubConfigSecret is the k8s secret information to use when authenticating via the GitHub API.
## You can choose to supply:
## A) a PAT token,

View File

@@ -8385,6 +8385,10 @@ spec:
type: object
runnerGroup:
type: string
runnerScaleSetLabels:
items:
type: string
type: array
runnerScaleSetName:
type: string
template:

View File

@@ -505,17 +505,35 @@ func (r *AutoscalingRunnerSetReconciler) createRunnerScaleSet(ctx context.Contex
}
if runnerScaleSet == nil {
labels := []scaleset.Label{
{
Name: autoscalingRunnerSet.Spec.RunnerScaleSetName,
Type: "System",
},
}
if labelCount := len(autoscalingRunnerSet.Spec.RunnerScaleSetLabels); labelCount > 0 {
unique := make(map[string]bool, labelCount+1)
unique[autoscalingRunnerSet.Spec.RunnerScaleSetName] = true
for _, label := range autoscalingRunnerSet.Spec.RunnerScaleSetLabels {
if _, exists := unique[label]; exists {
logger.Info("Duplicate label found. Skipping adding duplicate label to runner scale set", "label", label)
continue
}
labels = append(labels, scaleset.Label{
Name: label,
Type: "System",
})
unique[label] = true
}
}
runnerScaleSet, err = actionsClient.CreateRunnerScaleSet(
ctx,
&scaleset.RunnerScaleSet{
Name: autoscalingRunnerSet.Spec.RunnerScaleSetName,
RunnerGroupID: runnerGroupID,
Labels: []scaleset.Label{
{
Name: autoscalingRunnerSet.Spec.RunnerScaleSetName,
Type: "System",
},
},
Labels: labels,
RunnerSetting: scaleset.RunnerSetting{
DisableUpdate: true,
},

View File

@@ -17,9 +17,6 @@ ARC_NAME="arc"
ARC_NAMESPACE="arc-systems"
function install_arc() {
echo "Creating namespace ${ARC_NAMESPACE}"
kubectl create namespace "${SCALE_SET_NAMESPACE}"
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \

View File

@@ -17,9 +17,6 @@ ARC_NAME="arc"
ARC_NAMESPACE="arc-systems"
function install_arc() {
echo "Creating namespace ${ARC_NAMESPACE}"
kubectl create namespace "${SCALE_SET_NAMESPACE}"
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \

View File

@@ -22,9 +22,6 @@ function install_arc() {
return 1
}
echo "Creating namespace ${ARC_NAMESPACE}"
kubectl create namespace "${SCALE_SET_NAMESPACE}"
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \

View File

@@ -22,9 +22,6 @@ function install_arc() {
return 1
}
echo "Creating namespace ${ARC_NAMESPACE}"
kubectl create namespace "${SCALE_SET_NAMESPACE}"
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \

View File

@@ -0,0 +1,113 @@
#!/bin/bash
set -euo pipefail
DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
ROOT_DIR="$(realpath "${DIR}/../..")"
source "${DIR}/helper.sh"
VERSION="$(chart_version "${ROOT_DIR}/charts/gha-runner-scale-set-controller-experimental/Chart.yaml")" || exit 1
export VERSION
SCALE_SET_NAME="custom-label-$(date +'%M%S')$(((RANDOM + 100) % 100 + 1))"
SCALE_SET_NAMESPACE="arc-runners"
SCALE_SET_LABEL="custom-$(date +'%s')${RANDOM}"
WORKFLOW_FILE="arc-custom-label.yaml"
ARC_NAME="arc"
ARC_NAMESPACE="arc-systems"
function install_arc() {
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \
--create-namespace \
--set controller.manager.container.image="${IMAGE_NAME}:${IMAGE_TAG}" \
"${ROOT_DIR}/charts/gha-runner-scale-set-controller-experimental" \
--debug
if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then
NAMESPACE="${ARC_NAMESPACE}" log_arc
return 1
fi
}
function install_scale_set() {
echo "Installing scale set ${SCALE_SET_NAMESPACE}/${SCALE_SET_NAME} with label ${SCALE_SET_LABEL}"
helm install "${SCALE_SET_NAME}" \
--namespace "${SCALE_SET_NAMESPACE}" \
--create-namespace \
--set controllerServiceAccount.name="${ARC_NAME}-gha-rs-controller" \
--set controllerServiceAccount.namespace="${ARC_NAMESPACE}" \
--set auth.url="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \
--set auth.githubToken="${GITHUB_TOKEN}" \
--set scaleset.labels[0]="${SCALE_SET_LABEL}" \
"${ROOT_DIR}/charts/gha-runner-scale-set-experimental" \
--version="${VERSION}"
if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then
NAMESPACE="${ARC_NAMESPACE}" log_arc
return 1
fi
}
function verify_scale_set_label() {
local actual_label
actual_label="$(kubectl get autoscalingrunnersets.actions.github.com -n "${SCALE_SET_NAMESPACE}" -l app.kubernetes.io/instance="${SCALE_SET_NAME}" -o jsonpath='{.items[0].spec.runnerScaleSetLabels[0]}')"
if [[ "${actual_label}" != "${SCALE_SET_LABEL}" ]]; then
echo "Expected scale set label '${SCALE_SET_LABEL}', got '${actual_label}'" >&2
return 1
fi
}
function run_custom_label_workflow() {
local repo="${TARGET_ORG}/${TARGET_REPO}"
local queue_time
queue_time="$(date -u +%FT%TZ)"
gh workflow run -R "${repo}" "${WORKFLOW_FILE}" \
-f scaleset-label="${SCALE_SET_LABEL}" || return 1
local count=0
local run_id=
while true; do
if [[ "${count}" -ge 12 ]]; then
echo "Timeout waiting for custom label workflow to start" >&2
return 1
fi
run_id="$(gh run list -R "${repo}" --workflow "${WORKFLOW_FILE}" --created ">${queue_time}" --json databaseId --jq '.[0].databaseId' | head -n1)"
if [[ -n "${run_id}" ]]; then
break
fi
sleep 5
count=$((count + 1))
done
gh run watch "${run_id}" -R "${repo}" --exit-status
}
function main() {
local failed=()
build_image
create_cluster
install_arc
install_scale_set
verify_scale_set_label || failed+=("verify_scale_set_label")
run_custom_label_workflow || failed+=("run_custom_label_workflow")
INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set")
NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc")
delete_cluster
print_results "${failed[@]}"
}
main

View File

@@ -0,0 +1,112 @@
#!/bin/bash
set -euo pipefail
DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
ROOT_DIR="$(realpath "${DIR}/../..")"
source "${DIR}/helper.sh"
export VERSION="$(chart_version "${ROOT_DIR}/charts/gha-runner-scale-set-controller/Chart.yaml")"
SCALE_SET_NAME="custom-label-$(date +'%M%S')$(((RANDOM + 100) % 100 + 1))"
SCALE_SET_NAMESPACE="arc-runners"
SCALE_SET_LABEL="custom-$(date +'%s')${RANDOM}"
WORKFLOW_FILE="arc-custom-label.yaml"
ARC_NAME="arc"
ARC_NAMESPACE="arc-systems"
function install_arc() {
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \
--create-namespace \
--set image.repository="${IMAGE_NAME}" \
--set image.tag="${IMAGE_TAG}" \
"${ROOT_DIR}/charts/gha-runner-scale-set-controller" \
--debug
if ! NAME="${ARC_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_arc; then
NAMESPACE="${ARC_NAMESPACE}" log_arc
return 1
fi
}
function install_scale_set() {
echo "Installing scale set ${SCALE_SET_NAMESPACE}/${SCALE_SET_NAME} with label ${SCALE_SET_LABEL}"
helm install "${SCALE_SET_NAME}" \
--namespace "${SCALE_SET_NAMESPACE}" \
--create-namespace \
--set githubConfigUrl="https://github.com/${TARGET_ORG}/${TARGET_REPO}" \
--set githubConfigSecret.github_token="${GITHUB_TOKEN}" \
--set scaleSetLabels[0]="${SCALE_SET_LABEL}" \
"${ROOT_DIR}/charts/gha-runner-scale-set" \
--version="${VERSION}" \
--debug
if ! NAME="${SCALE_SET_NAME}" NAMESPACE="${ARC_NAMESPACE}" wait_for_scale_set; then
NAMESPACE="${ARC_NAMESPACE}" log_arc
return 1
fi
}
function verify_scale_set_label() {
local actual_label
actual_label="$(kubectl get autoscalingrunnersets.actions.github.com -n "${SCALE_SET_NAMESPACE}" -l app.kubernetes.io/instance="${SCALE_SET_NAME}" -o jsonpath='{.items[0].spec.runnerScaleSetLabels[0]}')"
if [[ "${actual_label}" != "${SCALE_SET_LABEL}" ]]; then
echo "Expected scale set label '${SCALE_SET_LABEL}', got '${actual_label}'" >&2
return 1
fi
}
function run_custom_label_workflow() {
local repo="${TARGET_ORG}/${TARGET_REPO}"
local queue_time
queue_time="$(date -u +%FT%TZ)"
gh workflow run -R "${repo}" "${WORKFLOW_FILE}" \
-f scaleset-label="${SCALE_SET_LABEL}" || return 1
local count=0
local run_id=
while true; do
if [[ "${count}" -ge 12 ]]; then
echo "Timeout waiting for custom label workflow to start" >&2
return 1
fi
run_id="$(gh run list -R "${repo}" --workflow "${WORKFLOW_FILE}" --created ">${queue_time}" --json databaseId --jq '.[0].databaseId' | head -n1)"
if [[ -n "${run_id}" ]]; then
break
fi
sleep 5
count=$((count + 1))
done
gh run watch "${run_id}" -R "${repo}" --exit-status
}
function main() {
local failed=()
build_image
create_cluster
install_arc
install_scale_set
verify_scale_set_label || failed+=("verify_scale_set_label")
run_custom_label_workflow || failed+=("run_custom_label_workflow")
INSTALLATION_NAME="${SCALE_SET_NAME}" NAMESPACE="${SCALE_SET_NAMESPACE}" cleanup_scale_set || failed+=("cleanup_scale_set")
NAMESPACE="${ARC_NAMESPACE}" log_arc || failed+=("log_arc")
delete_cluster
print_results "${failed[@]}"
}
main

View File

@@ -18,9 +18,6 @@ ARC_NAME="arc"
ARC_NAMESPACE="arc-systems"
function install_arc() {
echo "Creating namespace ${ARC_NAMESPACE}"
kubectl create namespace "${SCALE_SET_NAMESPACE}"
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \

View File

@@ -17,9 +17,6 @@ ARC_NAME="arc"
ARC_NAMESPACE="arc-systems"
function install_arc() {
echo "Creating namespace ${ARC_NAMESPACE}"
kubectl create namespace "${SCALE_SET_NAMESPACE}"
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \

View File

@@ -17,9 +17,6 @@ ARC_NAME="arc"
ARC_NAMESPACE="arc-systems"
function install_arc() {
echo "Creating namespace ${ARC_NAMESPACE}"
kubectl create namespace "${SCALE_SET_NAMESPACE}"
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \

View File

@@ -17,9 +17,6 @@ ARC_NAME="arc"
ARC_NAMESPACE="arc-systems"
function install_arc() {
echo "Creating namespace ${ARC_NAMESPACE}"
kubectl create namespace "${SCALE_SET_NAMESPACE}"
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \

View File

@@ -22,9 +22,6 @@ function install_arc() {
return 1
}
echo "Creating namespace ${ARC_NAMESPACE}"
kubectl create namespace "${SCALE_SET_NAMESPACE}"
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \

View File

@@ -22,9 +22,6 @@ function install_arc() {
return 1
}
echo "Creating namespace ${ARC_NAMESPACE}"
kubectl create namespace "${SCALE_SET_NAMESPACE}"
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \

View File

@@ -17,9 +17,6 @@ ARC_NAME="arc"
ARC_NAMESPACE="${SCALE_SET_NAMESPACE}"
function install_arc() {
echo "Creating namespace ${ARC_NAMESPACE}"
kubectl create namespace "${SCALE_SET_NAMESPACE}"
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \

View File

@@ -17,9 +17,6 @@ ARC_NAME="arc"
ARC_NAMESPACE="${SCALE_SET_NAMESPACE}"
function install_arc() {
echo "Creating namespace ${ARC_NAMESPACE}"
kubectl create namespace "${SCALE_SET_NAMESPACE}"
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \

View File

@@ -21,7 +21,6 @@ ARC_NAMESPACE="arc-systems"
function install_arc() {
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \
--create-namespace \

View File

@@ -21,7 +21,6 @@ ARC_NAMESPACE="arc-systems"
function install_arc() {
echo "Installing ARC"
helm install "${ARC_NAME}" \
--namespace "${ARC_NAMESPACE}" \
--create-namespace \