Configurable "runner and DinD in a single container" (#126)

This commit is contained in:
Juho Saarinen
2020-10-20 02:48:28 +03:00
committed by GitHub
parent 7d0bfb77e3
commit 92920926fe
15 changed files with 330 additions and 78 deletions

View File

@@ -43,6 +43,13 @@ jobs:
--tag ${DOCKERHUB_USERNAME}/actions-runner:v${RUNNER_VERSION} \
--tag ${DOCKERHUB_USERNAME}/actions-runner:latest \
-f Dockerfile .
docker buildx build \
--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
--platform linux/amd64,linux/arm64 \
--tag ${DOCKERHUB_USERNAME}/actions-runner-dind:v${RUNNER_VERSION} \
--tag ${DOCKERHUB_USERNAME}/actions-runner-dind:latest \
-f Dockerfile.dindrunner .
- name: Login to GitHub Docker Registry
run: echo "${DOCKERHUB_PASSWORD}" | docker login -u "${DOCKERHUB_USERNAME}" --password-stdin
@@ -61,4 +68,11 @@ jobs:
--platform linux/amd64,linux/arm64 \
--tag ${DOCKERHUB_USERNAME}/actions-runner:v${RUNNER_VERSION} \
--tag ${DOCKERHUB_USERNAME}/actions-runner:latest \
-f Dockerfile . --push
-f Dockerfile . --push
docker buildx build \
--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
--platform linux/amd64,linux/arm64 \
--tag ${DOCKERHUB_USERNAME}/actions-runner-dind:v${RUNNER_VERSION} \
--tag ${DOCKERHUB_USERNAME}/actions-runner-dind:latest \
-f Dockerfile.dindrunner . --push

View File

@@ -2,8 +2,7 @@ NAME ?= summerwind/actions-runner-controller
VERSION ?= latest
# From https://github.com/VictoriaMetrics/operator/pull/44
YAML_DROP=$(YQ) delete --inplace
TEMPLATE_DROP_PREFIX=spec.validation.openAPIV3Schema.properties.spec.properties.template.properties.spec.properties
CONTAINER_DROP_PREFIX=spec.validation.openAPIV3Schema.properties.spec.properties
YAML_DROP_PREFIX=spec.validation.openAPIV3Schema.properties.spec.properties
# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
CRD_OPTIONS ?= "crd:trivialVersions=true"
@@ -76,34 +75,21 @@ vet:
# workaround for CRD issue with k8s 1.18 & controller-gen
# ref: https://github.com/kubernetes/kubernetes/issues/91395
fix118: yq
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(TEMPLATE_DROP_PREFIX).containers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(TEMPLATE_DROP_PREFIX).initContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(TEMPLATE_DROP_PREFIX).sidecarContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(TEMPLATE_DROP_PREFIX).ephemeralContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(TEMPLATE_DROP_PREFIX).containers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(TEMPLATE_DROP_PREFIX).initContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(TEMPLATE_DROP_PREFIX).sidecarContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(TEMPLATE_DROP_PREFIX).ephemeralContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(TEMPLATE_DROP_PREFIX).containers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(TEMPLATE_DROP_PREFIX).initContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(TEMPLATE_DROP_PREFIX).sidecarContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(TEMPLATE_DROP_PREFIX).ephemeralContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(CONTAINER_DROP_PREFIX).initContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(CONTAINER_DROP_PREFIX).sidecarContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(CONTAINER_DROP_PREFIX).ephemeralContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(CONTAINER_DROP_PREFIX).containers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(CONTAINER_DROP_PREFIX).initContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(CONTAINER_DROP_PREFIX).sidecarContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(CONTAINER_DROP_PREFIX).ephemeralContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(CONTAINER_DROP_PREFIX).containers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(CONTAINER_DROP_PREFIX).initContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(CONTAINER_DROP_PREFIX).sidecarContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(CONTAINER_DROP_PREFIX).ephemeralContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.containers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.initContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.sidecarContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.ephemeralContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.containers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.initContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.sidecarContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.ephemeralContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).containers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).initContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).sidecarContainers.items.properties
$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).ephemeralContainers.items.properties
# Generate code
generate: generate-118 fix118
generate-118: controller-gen
generate: controller-gen
$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."
# Build the docker image
@@ -140,6 +126,7 @@ github-release: release
# download controller-gen if necessary
controller-gen:
ifeq (, $(shell which controller-gen))
ifeq (, $(wildcard $(GOBIN)/controller-gen))
@{ \
set -e ;\
CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
@@ -148,6 +135,7 @@ ifeq (, $(shell which controller-gen))
go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0 ;\
rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
}
endif
CONTROLLER_GEN=$(GOBIN)/controller-gen
else
CONTROLLER_GEN=$(shell which controller-gen)
@@ -155,18 +143,17 @@ endif
# find or download yq
# download yq if necessary
# Use always go-version to get consistent line wraps etc.
yq:
ifeq (, $(shell which yq))
ifeq (, $(wildcard $(GOBIN)/yq))
echo "Downloading yq"
@{ \
set -e ;\
YQ_TMP_DIR=$$(mktemp -d) ;\
cd $$YQ_TMP_DIR ;\
go mod init tmp ;\
go get github.com/mikefarah/yq/v3 ;\
go get github.com/mikefarah/yq/v3@3.4.0 ;\
rm -rf $$YQ_TMP_DIR ;\
}
YQ=$(GOBIN)/yq
else
YQ=$(shell which yq)
endif
YQ=$(GOBIN)/yq

View File

@@ -250,6 +250,27 @@ spec:
repositoryNames:
- summerwind/actions-runner-controller
```
## Runner with DinD
When using default runner, runner pod starts up 2 containers: runner and DinD (Docker-in-Docker). This might create issues if there's `LimitRange` set to namespace.
```yaml
# dindrunnerdeployment.yaml
apiVersion: actions.summerwind.dev/v1alpha1
kind: RunnerDeployment
metadata:
name: example-dindrunnerdeploy
spec:
replicas: 2
template:
spec:
image: summerwind/actions-runner-dind
dockerWithinRunnerContainer: true
repository: mumoshu/actions-runner-controller-ci
env: []
```
This also helps with resources, as you don't need to give resources separately to docker and runner.
## Additional tweaks
@@ -283,6 +304,17 @@ spec:
requests:
cpu: "2.0"
memory: "4Gi"
# If set to true, runner pod container only 1 container that's expected to be able to run docker, too.
# image summerwind/actions-runner-dind or custom one should be used with true -value
dockerWithinRunnerContainer: false
# Valid if dockerWithinRunnerContainer is not true
dockerdContainerResources:
limits:
cpu: "4.0"
memory: "8Gi"
requests:
cpu: "2.0"
memory: "4Gi"
sidecarContainers:
- name: mysql
image: mysql:5.7

View File

@@ -77,6 +77,8 @@ type RunnerSpec struct {
EphemeralContainers []corev1.EphemeralContainer `json:"ephemeralContainers,omitempty"`
// +optional
TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
// +optional
DockerWithinRunnerContainer *bool `json:"dockerWithinRunnerContainer,omitempty"`
}
// ValidateRepository validates repository field.

View File

@@ -524,6 +524,11 @@ func (in *RunnerSpec) DeepCopyInto(out *RunnerSpec) {
*out = new(int64)
**out = **in
}
if in.DockerWithinRunnerContainer != nil {
in, out := &in.DockerWithinRunnerContainer, &out.DockerWithinRunnerContainer
*out = new(bool)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerSpec.

View File

@@ -399,6 +399,8 @@ spec:
- name
type: object
type: array
dockerWithinRunnerContainer:
type: boolean
env:
items:
description: EnvVar represents an environment variable present in a Container.

View File

@@ -399,6 +399,8 @@ spec:
- name
type: object
type: array
dockerWithinRunnerContainer:
type: boolean
env:
items:
description: EnvVar represents an environment variable present in a Container.

View File

@@ -393,6 +393,8 @@ spec:
- name
type: object
type: array
dockerWithinRunnerContainer:
type: boolean
env:
items:
description: EnvVar represents an environment variable present in a Container.

View File

@@ -276,8 +276,8 @@ func (r *RunnerReconciler) unregisterRunner(ctx context.Context, org, repo, name
func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
var (
privileged bool = true
group int64 = 0
privileged bool = true
dockerdInRunner bool = runner.Spec.DockerWithinRunnerContainer != nil && *runner.Spec.DockerWithinRunnerContainer
)
runnerImage := runner.Spec.Image
@@ -311,6 +311,10 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
Name: "RUNNER_TOKEN",
Value: runner.Status.Registration.Token,
},
{
Name: "DOCKERD_IN_RUNNER",
Value: fmt.Sprintf("%v", dockerdInRunner),
},
}
env = append(env, runner.Spec.Env...)
@@ -330,56 +334,61 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
ImagePullPolicy: runnerImagePullPolicy,
Env: env,
EnvFrom: runner.Spec.EnvFrom,
VolumeMounts: []corev1.VolumeMount{
{
Name: "work",
MountPath: "/runner/_work",
},
{
Name: "docker",
MountPath: "/var/run",
},
},
SecurityContext: &corev1.SecurityContext{
RunAsGroup: &group,
// Runner need to run privileged if it contains DinD
Privileged: runner.Spec.DockerWithinRunnerContainer,
},
Resources: runner.Spec.Resources,
},
{
Name: "docker",
Image: r.DockerImage,
VolumeMounts: []corev1.VolumeMount{
{
Name: "work",
MountPath: "/runner/_work",
},
{
Name: "docker",
MountPath: "/var/run",
},
},
SecurityContext: &corev1.SecurityContext{
Privileged: &privileged,
},
},
},
Volumes: []corev1.Volume{
{
Name: "work",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
{
Name: "docker",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
},
},
}
if !dockerdInRunner {
pod.Spec.Volumes = []corev1.Volume{
{
Name: "work",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
{
Name: "docker",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
}
pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
{
Name: "work",
MountPath: "/runner/_work",
},
{
Name: "docker",
MountPath: "/var/run",
},
}
pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{
Name: "docker",
Image: r.DockerImage,
VolumeMounts: []corev1.VolumeMount{
{
Name: "work",
MountPath: "/runner/_work",
},
{
Name: "docker",
MountPath: "/var/run",
},
},
SecurityContext: &corev1.SecurityContext{
Privileged: &privileged,
},
})
}
if len(runner.Spec.Containers) != 0 {
pod.Spec.Containers = runner.Spec.Containers
for i := 0; i < len(pod.Spec.Containers); i++ {

View File

@@ -0,0 +1,99 @@
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive
# Dev + DinD dependencies
RUN apt update \
&& apt install -y software-properties-common \
&& add-apt-repository -y ppa:git-core/ppa \
&& apt install -y \
build-essential \
curl \
ca-certificates \
dnsutils \
ftp \
git \
iproute2 \
iptables \
iputils-ping \
jq \
libunwind8 \
locales \
netcat \
openssh-client \
parallel \
rsync \
shellcheck \
sudo \
supervisor \
telnet \
time \
tzdata \
unzip \
upx \
wget \
zip \
zstd \
&& rm -rf /var/lib/apt/list/*
# Runner user
RUN adduser --disabled-password --gecos "" --uid 1000 runner \
&& groupadd docker \
&& usermod -aG sudo runner \
&& usermod -aG docker runner \
&& echo "%sudo ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers
ARG TARGETPLATFORM
ARG RUNNER_VERSION=2.272.0
ARG DOCKER_CHANNEL=stable
ARG DOCKER_VERSION=19.03.13
ARG DEBUG=false
# Docker installation
RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
&& if [ "$ARCH" = "arm64" ]; then export ARCH=aarch64 ; fi \
&& if [ "$ARCH" = "amd64" ]; then export ARCH=x86_64 ; fi \
&& if ! curl -L -o docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${ARCH}/docker-${DOCKER_VERSION}.tgz"; then \
echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${ARCH}'"; \
exit 1; \
fi; \
tar --extract \
--file docker.tgz \
--strip-components 1 \
--directory /usr/local/bin/ \
; \
rm docker.tgz; \
dockerd --version; \
docker --version
# Runner download supports amd64 as x64
RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
&& if [ "$ARCH" = "amd64" ]; then export ARCH=x64 ; fi \
&& mkdir -p /runner \
&& cd /runner \
&& curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${ARCH}-${RUNNER_VERSION}.tar.gz \
&& tar xzf ./runner.tar.gz \
&& rm runner.tar.gz \
&& ./bin/installdependencies.sh \
&& rm -rf /var/lib/apt/lists/*
COPY modprobe startup.sh /usr/local/bin/
COPY supervisor/ /etc/supervisor/conf.d/
COPY logger.sh /opt/bash-utils/logger.sh
COPY entrypoint.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/startup.sh /usr/local/bin/entrypoint.sh /usr/local/bin/modprobe
RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
&& curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_${ARCH} \
&& chmod +x /usr/local/bin/dumb-init
VOLUME /var/lib/docker
COPY patched /runner/patched
# No group definition, as that makes it harder to run docker.
USER runner
ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
CMD ["startup.sh"]

View File

@@ -1,4 +1,5 @@
NAME ?= summerwind/actions-runner
DIND_RUNNER_NAME ?= ${NAME}-dind
TAG ?= latest
RUNNER_VERSION ?= 2.273.5
@@ -23,10 +24,14 @@ endif
docker-build:
docker build --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${NAME}:${TAG} -t ${NAME}:v${RUNNER_VERSION} .
docker build --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${DIND_RUNNER_NAME}:${TAG} -t ${DIND_RUNNER_NAME}:v${RUNNER_VERSION} -f Dockerfile.dindrunner .
docker-push:
docker push ${NAME}:${TAG}
docker push ${NAME}:v${RUNNER_VERSION}
docker push ${DIND_RUNNER_NAME}:${TAG}
docker push ${DIND_RUNNER_NAME}:v${RUNNER_VERSION}
docker-buildx:
export DOCKER_CLI_EXPERIMENTAL=enabled
@@ -39,3 +44,9 @@ docker-buildx:
-t "${NAME}:latest" \
-f Dockerfile \
. ${PUSH_ARG}
docker buildx build --platform ${PLATFORMS} \
--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
-t "${DIND_RUNNER_NAME}:latest" \
-f Dockerfile.dindrunner \
. ${PUSH_ARG}

24
runner/logger.sh Normal file
View File

@@ -0,0 +1,24 @@
#!/bin/sh
# Logger from this post http://www.cubicrace.com/2016/03/log-tracing-mechnism-for-shell-scripts.html
function INFO(){
local function_name="${FUNCNAME[1]}"
local msg="$1"
timeAndDate=`date`
echo "[$timeAndDate] [INFO] [${0}] $msg"
}
function DEBUG(){
local function_name="${FUNCNAME[1]}"
local msg="$1"
timeAndDate=`date`
echo "[$timeAndDate] [DEBUG] [${0}] $msg"
}
function ERROR(){
local function_name="${FUNCNAME[1]}"
local msg="$1"
timeAndDate=`date`
echo "[$timeAndDate] [ERROR] $msg"
}

20
runner/modprobe Normal file
View File

@@ -0,0 +1,20 @@
#!/bin/sh
set -eu
# "modprobe" without modprobe
# https://twitter.com/lucabruno/status/902934379835662336
# this isn't 100% fool-proof, but it'll have a much higher success rate than simply using the "real" modprobe
# Docker often uses "modprobe -va foo bar baz"
# so we ignore modules that start with "-"
for module; do
if [ "${module#-}" = "$module" ]; then
ip link show "$module" || true
lsmod | grep "$module" || true
fi
done
# remove /usr/local/... from PATH so we can exec the real modprobe as a last resort
export PATH='/usr/sbin:/usr/bin:/sbin:/bin'
exec modprobe "$@"

37
runner/startup.sh Normal file
View File

@@ -0,0 +1,37 @@
#!/bin/bash
source /opt/bash-utils/logger.sh
function wait_for_process () {
local max_time_wait=30
local process_name="$1"
local waited_sec=0
while ! pgrep "$process_name" >/dev/null && ((waited_sec < max_time_wait)); do
INFO "Process $process_name is not running yet. Retrying in 1 seconds"
INFO "Waited $waited_sec seconds of $max_time_wait seconds"
sleep 1
((waited_sec=waited_sec+1))
if ((waited_sec >= max_time_wait)); then
return 1
fi
done
return 0
}
INFO "Starting supervisor"
sudo /usr/bin/supervisord -n >> /dev/null 2>&1 &
INFO "Waiting for processes to be running"
processes=(dockerd)
for process in "${processes[@]}"; do
wait_for_process "$process"
if [ $? -ne 0 ]; then
ERROR "$process is not running after max time"
exit 1
else
INFO "$process is running"
fi
done
# Wait processes to be running
entrypoint.sh

View File

@@ -0,0 +1,6 @@
[program:dockerd]
command=/usr/local/bin/dockerd
autostart=true
autorestart=true
stderr_logfile=/var/log/dockerd.err.log
stdout_logfile=/var/log/dockerd.out.log