diff --git a/packages/k8s/src/hooks/run-container-step.ts b/packages/k8s/src/hooks/run-container-step.ts index 891e618..9cec87d 100644 --- a/packages/k8s/src/hooks/run-container-step.ts +++ b/packages/k8s/src/hooks/run-container-step.ts @@ -1,5 +1,4 @@ import * as core from '@actions/core' -import { v4 as uuidv4 } from 'uuid' import * as k8s from '@kubernetes/client-node' import { RunContainerStepArgs } from 'hooklib' import { @@ -27,7 +26,6 @@ export async function runContainerStep( if (stepContainer.dockerfile) { const imagePath = `${generateBuildHandle()}/${generateBuildTag()}` const imageUrl = await containerBuild(stepContainer, imagePath) - // throw new Error('Building container actions is not currently supported') stepContainer.image = imageUrl } diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index d60d705..d0f6fe5 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -10,23 +10,18 @@ import { getVolumeClaimName, RunnerInstanceLabel } from '../hooks/constants' -import { - registryConfigMap, - registrySecret, - registryStatefulSet, - registryService, - kanikoPod -} from './kaniko' +import { kanikoPod } from './kaniko' import { PodPhase } from './utils' +import { + namespace, + kc, + k8sApi, + k8sBatchV1Api, + k8sAuthorizationV1Api, + registryNodePort +} from './settings' -const kc = new k8s.KubeConfig() - -kc.loadFromDefault() - -const k8sApi = kc.makeApiClient(k8s.CoreV1Api) -const k8sBatchV1Api = kc.makeApiClient(k8s.BatchV1Api) -const k8sAppsV1 = kc.makeApiClient(k8s.AppsV1Api) -const k8sAuthorizationV1Api = kc.makeApiClient(k8s.AuthorizationV1Api) +export * from './settings' export const POD_VOLUME_NAME = 'work' @@ -489,41 +484,9 @@ export async function containerBuild( args: RunContainerStepArgs, imagePath: string ): Promise { - const cm = registryConfigMap() - const secret = registrySecret() - const ss = registryStatefulSet() - const svc = registryService() - const port = svc?.spec?.ports?.[0]?.nodePort - const registryUri = `localhost:${port}/${imagePath}` - const pod = kanikoPod(args.workingDirectory, imagePath) - await Promise.all([ - k8sApi.createNamespacedConfigMap(namespace(), cm), - k8sApi.createNamespacedSecret(namespace(), secret) - ]) - try { - await k8sAppsV1.createNamespacedStatefulSet(namespace(), ss) - await waitForPodPhases( - 'docker-registry-0', - new Set([PodPhase.RUNNING]), - new Set([PodPhase.PENDING, PodPhase.UNKNOWN]) - ) - } catch (err) { - console.log(err) - console.log(JSON.stringify(err)) - throw err - } - try { - await k8sApi.createNamespacedService(namespace(), svc) - } catch (err) { - console.log(JSON.stringify(err)) - throw err - } - try { - await k8sApi.createNamespacedPod(namespace(), pod) - } catch (err) { - console.log(JSON.stringify(err)) - throw err - } + const registryUri = `localhost:${registryNodePort()}/${imagePath}` + const pod = kanikoPod(args.dockerfile, imagePath) + await k8sApi.createNamespacedPod(namespace(), pod) return registryUri } @@ -536,19 +499,6 @@ async function getCurrentNodeName(): Promise { } return nodeName } -export function namespace(): string { - if (process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']) { - return process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] - } - - const context = kc.getContexts().find(ctx => ctx.namespace) - if (!context?.namespace) { - throw new Error( - 'Failed to determine namespace, falling back to `default`. Namespace should be set in context, or in env variable "ACTIONS_RUNNER_KUBERNETES_NAMESPACE"' - ) - } - return context.namespace -} class BackOffManager { private backOffSeconds = 1 diff --git a/packages/k8s/src/k8s/kaniko.ts b/packages/k8s/src/k8s/kaniko.ts index dce10e8..0050fb0 100644 --- a/packages/k8s/src/k8s/kaniko.ts +++ b/packages/k8s/src/k8s/kaniko.ts @@ -1,201 +1,52 @@ import * as k8s from '@kubernetes/client-node' -import { getVolumeClaimName } from '../hooks/constants' +import * as path from 'path' +import { namespace, registryHost, registryPort } from './settings' +import { + getRunnerPodName, + getVolumeClaimName, + MAX_POD_NAME_LENGTH, + RunnerInstanceLabel +} from '../hooks/constants' import { POD_VOLUME_NAME } from '.' -const REGISTRY_CONFIG_MAP_YAML = ` -storage: - filesystem: - rootdirectory: /var/lib/registry - maxthreads: 100 -health: - storagedriver: - enabled: true - interval: 10s - threshold: 3 -http: - addr: :5000 - headers: - X-Content-Type-Options: - - nosniff -log: - fields: - service: registry -storage: - cache: - blobdescriptor: inmemory -version: 0.1 -`.trim() +export const KANIKO_MOUNT_PATH = '/mnt/kaniko' -export function registryConfigMap(): k8s.V1ConfigMap { - const cm = new k8s.V1ConfigMap() - cm.apiVersion = 'v1' - cm.data = { - 'config.yaml': REGISTRY_CONFIG_MAP_YAML - } - cm.kind = 'ConfigMap' - cm.metadata = new k8s.V1ObjectMeta() - cm.metadata.labels = { app: 'docker-registry' } - cm.metadata.name = 'docker-registry-config' - // TODO: make this configurable - - return cm -} - -export function registrySecret(): k8s.V1Secret { - const secret = new k8s.V1Secret() - secret.apiVersion = 'v1' - secret.data = { haSharedSecret: 'U29tZVZlcnlTdHJpbmdTZWNyZXQK' } - secret.kind = 'Secret' - secret.metadata = new k8s.V1ObjectMeta() - secret.metadata.labels = { - app: 'docker-registry', - chart: 'docker-registry-1.4.3' - } - secret.metadata.name = 'docker-registry-secret' - secret.type = 'Opaque' - - return secret -} - -export function registryStatefulSet(): k8s.V1StatefulSet { - const ss = new k8s.V1StatefulSet() - ss.apiVersion = 'apps/v1' - ss.metadata = new k8s.V1ObjectMeta() - ss.metadata.name = 'docker-registry' - - const spec = new k8s.V1StatefulSetSpec() - spec.selector = new k8s.V1LabelSelector() - spec.selector.matchLabels = { app: 'docker-registry' } - spec.serviceName = 'registry' - spec.replicas = 1 - - const tmpl = new k8s.V1PodTemplateSpec() - tmpl.metadata = new k8s.V1ObjectMeta() - tmpl.metadata.labels = { app: 'docker-registry' } - tmpl.spec = new k8s.V1PodSpec() - tmpl.spec.terminationGracePeriodSeconds = 5 // TODO: figure out for how long - - const c = new k8s.V1Container() - c.command = ['/bin/registry', 'serve', '/etc/docker/registry/config.yaml'] - c.env = [ - { - name: 'REGISTRY_HTTP_SECRET', - valueFrom: { - secretKeyRef: { - key: 'haSharedSecret', - name: 'docker-registry-secret' - } - } - }, - { - name: 'REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY', - value: '/var/lib/registry' - } - ] - c.image = 'registry:2.6.2' - c.name = 'docker-registry' - c.imagePullPolicy = 'IfNotPresent' - c.ports = [ - { - containerPort: 5000, - protocol: 'TCP' - } - ] - - c.volumeMounts = [ - { - mountPath: '/etc/docker/registry', - name: 'docker-registry-config' - } - ] - - c.livenessProbe = new k8s.V1Probe() - c.livenessProbe.failureThreshold = 3 - c.livenessProbe.periodSeconds = 10 - c.livenessProbe.successThreshold = 1 - c.livenessProbe.timeoutSeconds = 1 - c.livenessProbe.httpGet = new k8s.V1HTTPGetAction() - c.livenessProbe.httpGet.path = '/' - c.livenessProbe.httpGet.port = 5000 - c.livenessProbe.httpGet.scheme = 'HTTP' - - c.readinessProbe = new k8s.V1Probe() - c.readinessProbe.failureThreshold = 3 - c.readinessProbe.periodSeconds = 10 - c.readinessProbe.successThreshold = 1 - c.readinessProbe.timeoutSeconds = 1 - c.readinessProbe.httpGet = new k8s.V1HTTPGetAction() - c.readinessProbe.httpGet.path = '/' - c.readinessProbe.httpGet.port = 5000 - c.readinessProbe.httpGet.scheme = 'HTTP' - - tmpl.spec.containers = [c] - tmpl.spec.volumes = [ - { - name: 'docker-registry-config', - configMap: { - name: 'docker-registry-config' - } - } - ] - - spec.template = tmpl - ss.spec = spec - - return ss -} - -export function registryService(): k8s.V1Service { - const svc = new k8s.V1Service() - svc.apiVersion = 'v1' - svc.kind = 'Service' - svc.metadata = new k8s.V1ObjectMeta() - svc.metadata.name = 'docker-registry' - svc.metadata.labels = { - app: 'docker-registry' - } - const spec = new k8s.V1ServiceSpec() - spec.externalTrafficPolicy = 'Cluster' - spec.ports = [ - { - name: 'registry', - nodePort: 31500, - port: 5000, - protocol: 'TCP', - targetPort: 5000 - } - ] - spec.selector = { - app: 'docker-registry' - } - spec.sessionAffinity = 'None' - spec.type = 'NodePort' - svc.spec = spec - - return svc +function getKanikoName(): string { + return `${getRunnerPodName().substring( + 0, + MAX_POD_NAME_LENGTH - '-kaniko'.length + )}-kaniko` } export function kanikoPod( - workingDirectory: string, // git://github.com// + dockerfile: string, imagePath: string // /: ): k8s.V1Pod { const pod = new k8s.V1Pod() pod.apiVersion = 'v1' pod.kind = 'Pod' pod.metadata = new k8s.V1ObjectMeta() - pod.metadata.name = 'kaniko' + pod.metadata.name = getKanikoName() + const instanceLabel = new RunnerInstanceLabel() + pod.metadata.labels = { + [instanceLabel.key]: instanceLabel.value + } const spec = new k8s.V1PodSpec() const c = new k8s.V1Container() c.image = 'gcr.io/kaniko-project/executor:latest' c.name = 'kaniko' c.imagePullPolicy = 'Always' + const prefix = (process.env.RUNNER_WORKSPACE as string).split('_work')[0] + const subPath = path + .dirname(dockerfile) + .substring(prefix.length + '_work/'.length) + c.volumeMounts = [ { name: POD_VOLUME_NAME, - mountPath: '/mnt/kan', - subPath: - '_actions/fhammerl/container-actions-demo/main/docker-built-from-file', + mountPath: KANIKO_MOUNT_PATH, + subPath, readOnly: true } ] @@ -206,9 +57,9 @@ export function kanikoPod( } ] c.args = [ - '--dockerfile=Dockerfile', - `--context=dir:///mnt/kan`, - `--destination=docker-registry.default.svc.cluster.local:5000/${imagePath}` + `--dockerfile=${path.basename(dockerfile)}`, + `--context=dir://${KANIKO_MOUNT_PATH}`, + `--destination=${registryHost()}.${namespace()}.svc.cluster.local:${registryPort()}/${imagePath}` ] spec.containers = [c] spec.dnsPolicy = 'ClusterFirst' diff --git a/packages/k8s/src/k8s/settings.ts b/packages/k8s/src/k8s/settings.ts new file mode 100644 index 0000000..e05fa2d --- /dev/null +++ b/packages/k8s/src/k8s/settings.ts @@ -0,0 +1,47 @@ +import * as k8s from '@kubernetes/client-node' +export const kc = new k8s.KubeConfig() + +kc.loadFromDefault() + +export const k8sApi = kc.makeApiClient(k8s.CoreV1Api) +export const k8sBatchV1Api = kc.makeApiClient(k8s.BatchV1Api) +export const k8sAuthorizationV1Api = kc.makeApiClient(k8s.AuthorizationV1Api) + +export const POD_VOLUME_NAME = 'work' +export function namespace(): string { + if (process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']) { + return process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] + } + + const context = kc.getContexts().find(ctx => ctx.namespace) + if (!context?.namespace) { + throw new Error( + 'Failed to determine namespace, falling back to `default`. Namespace should be set in context, or in env variable "ACTIONS_RUNNER_KUBERNETES_NAMESPACE"' + ) + } + return context.namespace +} + +export function registryHost(): string { + const name = 'RUNNER_CONTAINER_HOOKS_REGISTRY_HOST' + if (process.env[name]) { + return process.env[name] + } + throw new Error(`environment variable ${name} is not set`) +} + +export function registryPort(): number { + const name = 'RUNNER_CONTAINER_HOOKS_REGISTRY_PORT' + if (process.env[name]) { + return parseInt(process.env[name]) + } + throw new Error(`environment variable ${name} is not set`) +} + +export function registryNodePort(): number { + const name = 'RUNNER_CONTAINER_HOOKS_REGISTRY_NODE_PORT' + if (process.env[name]) { + return parseInt(process.env[name]) + } + throw new Error(`environment variable ${name} is not set`) +} diff --git a/packages/k8s/tests/build-container-test.ts b/packages/k8s/tests/build-container-test.ts deleted file mode 100644 index 3e530ae..0000000 --- a/packages/k8s/tests/build-container-test.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { containerBuild } from '../src/k8s' - -jest.useRealTimers() - -describe('container build', () => { - beforeAll(async () => { - process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default' - }) - - it('should finish without throwing an exception', async () => { - await expect( - containerBuild( - { - workingDirectory: 'git://github.com/nikola-jokic/dockeraction.git' - }, - 'randhandle/randimg:123123' - ) - ).resolves.not.toThrow() - }) -}) diff --git a/packages/k8s/tests/run-container-step-test.ts b/packages/k8s/tests/run-container-step-test.ts index 108131b..49861a6 100644 --- a/packages/k8s/tests/run-container-step-test.ts +++ b/packages/k8s/tests/run-container-step-test.ts @@ -3,11 +3,10 @@ import { TestHelper } from './test-setup' jest.useRealTimers() -let testHelper: TestHelper +describe('Run container step with image', () => { + let testHelper: TestHelper + let runContainerStepData: any -let runContainerStepData: any - -describe('Run container step', () => { beforeEach(async () => { testHelper = new TestHelper() await testHelper.initialize() @@ -39,3 +38,33 @@ describe('Run container step', () => { ).resolves.not.toThrow() }) }) + +describe('run container step with docker build', () => { + let testHelper: TestHelper + let runContainerStepData: any + beforeEach(async () => { + testHelper = new TestHelper() + await testHelper.initialize() + runContainerStepData = testHelper.getRunContainerStepDefinition() + }) + + afterEach(async () => { + await testHelper.cleanup() + }) + + it('should build container and execute docker action', async () => { + const { registryName, registryPort, nodePort } = + await testHelper.createContainerRegistry() + + // process.env.RUNNER_CONTAINER_HOOKS_REGISTRY_HOST = 'docker-registry' + // process.env.RUNNER_CONTAINER_HOOKS_REGISTRY_PORT = '5000' + // process.env.RUNNER_CONTAINER_HOOKS_REGISTRY_NODE_PORT = '31500' + process.env.RUNNER_CONTAINER_HOOKS_REGISTRY_HOST = registryName + process.env.RUNNER_CONTAINER_HOOKS_REGISTRY_PORT = registryPort.toString() + process.env.RUNNER_CONTAINER_HOOKS_REGISTRY_NODE_PORT = nodePort.toString() + const actionPath = testHelper.initializeDockerAction() + const data = JSON.parse(JSON.stringify(runContainerStepData)) + data.args.dockerfile = `${actionPath}/Dockerfile` + await expect(runContainerStep(data.args)).resolves.not.toThrow() + }) +}) diff --git a/packages/k8s/tests/test-setup.ts b/packages/k8s/tests/test-setup.ts index 25efcf5..9e94d1f 100644 --- a/packages/k8s/tests/test-setup.ts +++ b/packages/k8s/tests/test-setup.ts @@ -2,7 +2,10 @@ import * as k8s from '@kubernetes/client-node' import * as fs from 'fs' import { HookData } from 'hooklib/lib' import * as path from 'path' +import internal from 'stream' import { v4 as uuidv4 } from 'uuid' +import { waitForPodPhases } from '../src/k8s' +import { PodPhase } from '../src/k8s/utils' const kc = new k8s.KubeConfig() @@ -10,6 +13,7 @@ kc.loadFromDefault() const k8sApi = kc.makeApiClient(k8s.CoreV1Api) const k8sStorageApi = kc.makeApiClient(k8s.StorageV1Api) +const k8sAppsV1 = kc.makeApiClient(k8s.AppsV1Api) export class TestHelper { private tempDirPath: string @@ -75,9 +79,9 @@ export class TestHelper { ) .catch(e => {}) } - public createFile(fileName?: string): string { + public createFile(fileName?: string, content = ''): string { const filePath = `${this.tempDirPath}/${fileName || uuidv4()}` - fs.writeFileSync(filePath, '') + fs.writeFileSync(filePath, content) return filePath } @@ -193,4 +197,237 @@ export class TestHelper { runContainerStep.args.registry = null return runContainerStep } + + public async createContainerRegistry(): Promise<{ + registryName: string + registryPort: number + nodePort: number + }> { + const registryName = 'docker-registry' + const registryPort = 5000 + const nodePort = 31500 + + const cm = registryConfigMap(registryName, registryPort) + const secret = registrySecret(registryName) + const ss = registryStatefulSet(registryName, registryPort) + const svc = registryService(registryName, registryPort, nodePort) + const namespace = + process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] || 'default' + + await Promise.all([ + k8sApi.createNamespacedConfigMap(namespace, cm), + k8sApi.createNamespacedSecret(namespace, secret) + ]) + await k8sAppsV1.createNamespacedStatefulSet(namespace, ss) + await waitForPodPhases( + `${registryName}-0`, + new Set([PodPhase.RUNNING]), + new Set([PodPhase.PENDING, PodPhase.UNKNOWN]) + ) + await k8sApi.createNamespacedService(namespace, svc) + return { + registryName, + registryPort, + nodePort + } + } + + public initializeDockerAction(): string { + const actionPath = `${this.tempDirPath}/_work/_actions/example-handle/example-repo/example-branch/mock-directory` + fs.mkdirSync(actionPath, { recursive: true }) + this.writeDockerfile(actionPath) + this.writeEntrypoint(actionPath) + return actionPath + } + + private writeDockerfile(actionPath: string) { + const content = `FROM ubuntu:latest +COPY entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"]` + fs.writeFileSync(`${actionPath}/Dockerfile`, content) + } + + private writeEntrypoint(actionPath) { + const content = `#!/bin/sh -l +echo "Hello $1" +time=$(date) +echo "::set-output name=time::$time"` + const entryPointPath = `${actionPath}/entrypoint.sh` + fs.writeFileSync(entryPointPath, content) + fs.chmodSync(entryPointPath, 0o755) + } +} + +function registryConfigMap(name: string, port: number): k8s.V1ConfigMap { + const REGISTRY_CONFIG_MAP_YAML = ` +storage: + filesystem: + rootdirectory: /var/lib/registry + maxthreads: 100 +health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 +http: + addr: :${port} + headers: + X-Content-Type-Options: + - nosniff +log: + fields: + service: registry +storage: + cache: + blobdescriptor: inmemory +version: 0.1 +`.trim() + const cm = new k8s.V1ConfigMap() + cm.apiVersion = 'v1' + cm.data = { + 'config.yaml': REGISTRY_CONFIG_MAP_YAML + } + cm.kind = 'ConfigMap' + cm.metadata = new k8s.V1ObjectMeta() + cm.metadata.labels = { app: name } + cm.metadata.name = `${name}-config` + + return cm +} + +function registryStatefulSet(name: string, port: number): k8s.V1StatefulSet { + const ss = new k8s.V1StatefulSet() + ss.apiVersion = 'apps/v1' + ss.metadata = new k8s.V1ObjectMeta() + ss.metadata.name = name + + const spec = new k8s.V1StatefulSetSpec() + spec.selector = new k8s.V1LabelSelector() + spec.selector.matchLabels = { app: 'docker-registry' } + spec.serviceName = 'registry' + spec.replicas = 1 + + const tmpl = new k8s.V1PodTemplateSpec() + tmpl.metadata = new k8s.V1ObjectMeta() + tmpl.metadata.labels = { app: name } + tmpl.spec = new k8s.V1PodSpec() + tmpl.spec.terminationGracePeriodSeconds = 5 // TODO: figure out for how long + + const c = new k8s.V1Container() + c.command = ['/bin/registry', 'serve', '/etc/docker/registry/config.yaml'] + c.env = [ + { + name: 'REGISTRY_HTTP_SECRET', + valueFrom: { + secretKeyRef: { + key: 'haSharedSecret', + name: `${name}-secret` + } + } + }, + { + name: 'REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY', + value: '/var/lib/registry' + } + ] + c.image = 'registry:2.6.2' + c.name = name + c.imagePullPolicy = 'IfNotPresent' + c.ports = [ + { + containerPort: port, + protocol: 'TCP' + } + ] + + c.volumeMounts = [ + { + mountPath: '/etc/docker/registry', + name: 'docker-registry-config' + } + ] + + c.livenessProbe = new k8s.V1Probe() + c.livenessProbe.failureThreshold = 3 + c.livenessProbe.periodSeconds = 10 + c.livenessProbe.successThreshold = 1 + c.livenessProbe.timeoutSeconds = 1 + c.livenessProbe.httpGet = new k8s.V1HTTPGetAction() + c.livenessProbe.httpGet.path = '/' + c.livenessProbe.httpGet.port = port + c.livenessProbe.httpGet.scheme = 'HTTP' + + c.readinessProbe = new k8s.V1Probe() + c.readinessProbe.failureThreshold = 3 + c.readinessProbe.periodSeconds = 10 + c.readinessProbe.successThreshold = 1 + c.readinessProbe.timeoutSeconds = 1 + c.readinessProbe.httpGet = new k8s.V1HTTPGetAction() + c.readinessProbe.httpGet.path = '/' + c.readinessProbe.httpGet.port = port + c.readinessProbe.httpGet.scheme = 'HTTP' + + tmpl.spec.containers = [c] + tmpl.spec.volumes = [ + { + name: `${name}-config`, + configMap: { + name: `${name}-config` + } + } + ] + + spec.template = tmpl + ss.spec = spec + + return ss +} +function registryService( + name: string, + port: number, + nodePort: number +): k8s.V1Service { + const svc = new k8s.V1Service() + svc.apiVersion = 'v1' + svc.kind = 'Service' + svc.metadata = new k8s.V1ObjectMeta() + svc.metadata.name = name + svc.metadata.labels = { + app: name + } + const spec = new k8s.V1ServiceSpec() + spec.externalTrafficPolicy = 'Cluster' + spec.ports = [ + { + name: 'registry', + nodePort: nodePort, + port: port, + protocol: 'TCP', + targetPort: port + } + ] + spec.selector = { + app: name + } + spec.sessionAffinity = 'None' + spec.type = 'NodePort' + svc.spec = spec + + return svc +} + +function registrySecret(name: string): k8s.V1Secret { + const secret = new k8s.V1Secret() + secret.apiVersion = 'v1' + secret.data = { haSharedSecret: 'U29tZVZlcnlTdHJpbmdTZWNyZXQK' } + secret.kind = 'Secret' + secret.metadata = new k8s.V1ObjectMeta() + secret.metadata.labels = { + app: name, + chart: `${name}-1.4.3` + } + secret.metadata.name = `${name}-secret` + secret.type = 'Opaque' + + return secret }