feat: allow for modifying var-run mount maximum size limit (#2624)

TROUBLESHOOTING.md

@@ -304,3 +304,27 @@ If you noticed that it takes several minutes for sidecar dind container to be cr
 **Solution**
 
 The solution is to switch to using faster storage, if you are experiencing this issue you are probably using HDD storage. Switching to SSD storage fixed the problem in my case. Most cloud providers have a list of storage options to use just pick something faster that your current disk, for on prem clusters you will need to invest in some SSDs.
+
+### Dockerd no space left on device
+
+**Problem**
+
+If you are running many containers on your runner you might encounter an issue where docker daemon is unable to start new containers and you see error `no space left on device`.  
+
+**Solution**
+
+Add a `dockerVarRunVolumeSizeLimit` key in your runner's spec with a higher size limit (the default is 1M) For instance:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: github-runner
+  namespace: github-system
+spec:
+  replicas: 6
+  template:
+    spec:
+      dockerVarRunVolumeSizeLimit: 50M
+      env: []
+```

apis/actions.summerwind.net/v1alpha1/runner_types.go

@@ -70,6 +70,8 @@ type RunnerConfig struct {
 	// +optional
 	DockerRegistryMirror *string `json:"dockerRegistryMirror,omitempty"`
 	// +optional
+	DockerVarRunVolumeSizeLimit *resource.Quantity `json:"dockerVarRunVolumeSizeLimit,omitempty"`
+	// +optional
 	VolumeSizeLimit *resource.Quantity `json:"volumeSizeLimit,omitempty"`
 	// +optional
 	VolumeStorageMedium *string `json:"volumeStorageMedium,omitempty"`

apis/actions.summerwind.net/v1alpha1/zz_generated.deepcopy.go

@@ -436,6 +436,11 @@ func (in *RunnerConfig) DeepCopyInto(out *RunnerConfig) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.DockerVarRunVolumeSizeLimit != nil {
+		in, out := &in.DockerVarRunVolumeSizeLimit, &out.DockerVarRunVolumeSizeLimit
+		x := (*in).DeepCopy()
+		*out = &x
+	}
 	if in.VolumeSizeLimit != nil {
 		in, out := &in.VolumeSizeLimit, &out.VolumeSizeLimit
 		x := (*in).DeepCopy()

charts/actions-runner-controller/crds/actions.summerwind.dev_runnerdeployments.yaml

@@ -1497,6 +1497,12 @@ spec:
                           type: integer
                         dockerRegistryMirror:
                           type: string
+                        dockerVarRunVolumeSizeLimit:
+                          anyOf:
+                            - type: integer
+                            - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
                         dockerVolumeMounts:
                           items:
                             description: VolumeMount describes a mounting of a Volume within a container.

charts/actions-runner-controller/crds/actions.summerwind.dev_runnerreplicasets.yaml

@@ -1479,6 +1479,12 @@ spec:
                           type: integer
                         dockerRegistryMirror:
                           type: string
+                        dockerVarRunVolumeSizeLimit:
+                          anyOf:
+                            - type: integer
+                            - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
                         dockerVolumeMounts:
                           items:
                             description: VolumeMount describes a mounting of a Volume within a container.

charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml

@@ -1432,6 +1432,12 @@ spec:
                   type: integer
                 dockerRegistryMirror:
                   type: string
+                dockerVarRunVolumeSizeLimit:
+                  anyOf:
+                    - type: integer
+                    - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
                 dockerVolumeMounts:
                   items:
                     description: VolumeMount describes a mounting of a Volume within a container.

charts/actions-runner-controller/crds/actions.summerwind.dev_runnersets.yaml

@@ -55,6 +55,12 @@ spec:
                   type: integer
                 dockerRegistryMirror:
                   type: string
+                dockerVarRunVolumeSizeLimit:
+                  anyOf:
+                    - type: integer
+                    - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
                 dockerdWithinRunnerContainer:
                   type: boolean
                 effectiveTime:

config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml

@@ -1497,6 +1497,12 @@ spec:
                           type: integer
                         dockerRegistryMirror:
                           type: string
+                        dockerVarRunVolumeSizeLimit:
+                          anyOf:
+                            - type: integer
+                            - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
                         dockerVolumeMounts:
                           items:
                             description: VolumeMount describes a mounting of a Volume within a container.

config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml

@@ -1479,6 +1479,12 @@ spec:
                           type: integer
                         dockerRegistryMirror:
                           type: string
+                        dockerVarRunVolumeSizeLimit:
+                          anyOf:
+                            - type: integer
+                            - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
                         dockerVolumeMounts:
                           items:
                             description: VolumeMount describes a mounting of a Volume within a container.

config/crd/bases/actions.summerwind.dev_runners.yaml

@@ -1432,6 +1432,12 @@ spec:
                   type: integer
                 dockerRegistryMirror:
                   type: string
+                dockerVarRunVolumeSizeLimit:
+                  anyOf:
+                    - type: integer
+                    - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
                 dockerVolumeMounts:
                   items:
                     description: VolumeMount describes a mounting of a Volume within a container.

config/crd/bases/actions.summerwind.dev_runnersets.yaml

@@ -55,6 +55,12 @@ spec:
                   type: integer
                 dockerRegistryMirror:
                   type: string
+                dockerVarRunVolumeSizeLimit:
+                  anyOf:
+                    - type: integer
+                    - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
                 dockerdWithinRunnerContainer:
                   type: boolean
                 effectiveTime:

controllers/actions.summerwind.net/runner_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"reflect"
 	"strconv"
 	"strings"
@@ -30,7 +31,6 @@ import (
 	"github.com/go-logr/logr"
 
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -810,6 +810,11 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru
 		dockerRegistryMirror = *runnerSpec.DockerRegistryMirror
 	}
 
+	if runnerSpec.DockerVarRunVolumeSizeLimit == nil {
+		runnerSpec.DockerVarRunVolumeSizeLimit = resource.NewScaledQuantity(1, resource.Mega)
+
+	}
+
 	// Be aware some of the environment variables are used
 	// in the runner entrypoint script
 	env := []corev1.EnvVar{
@@ -1080,7 +1085,7 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru
 				VolumeSource: corev1.VolumeSource{
 					EmptyDir: &corev1.EmptyDirVolumeSource{
 						Medium:    corev1.StorageMediumMemory,
-						SizeLimit: resource.NewScaledQuantity(1, resource.Mega),
+						SizeLimit: runnerSpec.DockerVarRunVolumeSizeLimit,
 					},
 				},
 			},