Merge branch 'main' of https://github.com/actions/runner-container-hooks into fhammerl+nikola-jokic/kaniko

wait for kaniko

packages/k8s/src/hooks/run-container-step.ts

@@ -8,7 +8,8 @@ import {
   getPodLogs,
   getPodStatus,
   waitForJobToComplete,
-  waitForPodPhases
+  waitForPodPhases,
+  containerBuild
 } from '../k8s'
 import {
   containerVolumes,
@@ -23,7 +24,8 @@ export async function runContainerStep(
   stepContainer: RunContainerStepArgs
 ): Promise<number> {
   if (stepContainer.dockerfile) {
-    throw new Error('Building container actions is not currently supported')
+    const imageUrl = await containerBuild(stepContainer)
+    stepContainer.image = imageUrl
   }
 
   let secretName: string | undefined = undefined

packages/k8s/src/k8s/index.ts

@@ -1,6 +1,6 @@
 import * as core from '@actions/core'
 import * as k8s from '@kubernetes/client-node'
-import { ContainerInfo, Registry } from 'hooklib'
+import { RunContainerStepArgs, ContainerInfo, Registry } from 'hooklib'
 import * as stream from 'stream'
 import {
   getJobPodName,
@@ -10,15 +10,25 @@ import {
   getVolumeClaimName,
   RunnerInstanceLabel
 } from '../hooks/constants'
+import { kanikoPod } from './kaniko'
+import { v4 as uuidv4 } from 'uuid'
 import { PodPhase } from './utils'
+import {
+  namespace,
+  kc,
+  k8sApi,
+  k8sBatchV1Api,
+  k8sAuthorizationV1Api,
+  localRegistryNodePort,
+  localRegistryHost,
+  localRegistryPort,
+  remoteRegistryHost,
+  remoteRegistryHandle,
+  remoteRegistrySecretName,
+  isLocalRegistrySet
+} from './settings'
 
-const kc = new k8s.KubeConfig()
+export * from './settings'
-
-kc.loadFromDefault()
-
-const k8sApi = kc.makeApiClient(k8s.CoreV1Api)
-const k8sBatchV1Api = kc.makeApiClient(k8s.BatchV1Api)
-const k8sAuthorizationV1Api = kc.makeApiClient(k8s.AuthorizationV1Api)
 
 export const POD_VOLUME_NAME = 'work'
 
@@ -46,12 +56,6 @@ export const requiredPermissions = [
     verbs: ['get', 'list', 'create', 'delete'],
     resource: 'jobs',
     subresource: ''
-  },
-  {
-    group: '',
-    verbs: ['create', 'delete', 'get', 'list'],
-    resource: 'secrets',
-    subresource: ''
   }
 ]
 
@@ -325,8 +329,20 @@ export async function waitForPodPhases(
   const backOffManager = new BackOffManager(maxTimeSeconds)
   let phase: PodPhase = PodPhase.UNKNOWN
   try {
-    while (true) {
+    let retryCount = 0
-      phase = await getPodPhase(podName)
+    while (retryCount < 3) {
+      try {
+        phase = await getPodPhase(podName)
+      } catch (err) {
+        const e = err as k8s.HttpError
+        if (e?.body?.reason === 'NotFound') {
+          retryCount++
+          await backOffManager.backOff()
+          continue
+        } else {
+          throw err
+        }
+      }
       if (awaitingPhases.has(phase)) {
         return
       }
@@ -338,6 +354,7 @@ export async function waitForPodPhases(
       }
       await backOffManager.backOff()
     }
+    throw new Error(`Failed to get pod phase after ${retryCount} attempts`)
   } catch (error) {
     throw new Error(`Pod ${podName} is unhealthy with phase status ${phase}`)
   }
@@ -464,6 +481,42 @@ export async function isPodContainerAlpine(
   return isAlpine
 }
 
+export async function containerBuild(
+  args: RunContainerStepArgs
+): Promise<string> {
+  let kanikoRegistry = ''
+  let pullRegistry = ''
+  let secretName: string | undefined = undefined
+  if (isLocalRegistrySet()) {
+    const host = `${localRegistryHost()}.${namespace()}.svc.cluster.local`
+    const port = localRegistryPort()
+    const uri = `${generateBuildHandle()}/${generateBuildImage()}`
+    kanikoRegistry = `${host}:${port}/${uri}`
+    pullRegistry = `localhost:${localRegistryNodePort()}/${uri}`
+  } else {
+    const uri = `${remoteRegistryHandle()}/${generateBuildImage()}`
+    if (remoteRegistryHost()) {
+      kanikoRegistry = `${remoteRegistryHost()}/${uri}`
+    } else {
+      kanikoRegistry = uri
+    }
+    pullRegistry = kanikoRegistry
+    secretName = remoteRegistrySecretName()
+  }
+
+  const pod = kanikoPod(args.dockerfile, kanikoRegistry, secretName)
+  if (!pod.metadata?.name) {
+    throw new Error('kaniko pod name is not set')
+  }
+  await k8sApi.createNamespacedPod(namespace(), pod)
+  await waitForPodPhases(
+    pod.metadata.name,
+    new Set([PodPhase.SUCCEEDED]),
+    new Set([PodPhase.PENDING, PodPhase.UNKNOWN, PodPhase.RUNNING])
+  )
+  return pullRegistry
+}
+
 async function getCurrentNodeName(): Promise<string> {
   const resp = await k8sApi.readNamespacedPod(getRunnerPodName(), namespace())
 
@@ -473,19 +526,6 @@ async function getCurrentNodeName(): Promise<string> {
   }
   return nodeName
 }
-export function namespace(): string {
-  if (process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']) {
-    return process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']
-  }
-
-  const context = kc.getContexts().find(ctx => ctx.namespace)
-  if (!context?.namespace) {
-    throw new Error(
-      'Failed to determine namespace, falling back to `default`. Namespace should be set in context, or in env variable "ACTIONS_RUNNER_KUBERNETES_NAMESPACE"'
-    )
-  }
-  return context.namespace
-}
 
 class BackOffManager {
   private backOffSeconds = 1
@@ -551,3 +591,11 @@ export function containerPorts(
   }
   return ports
 }
+
+function generateBuildImage(): string {
+  return `${uuidv4()}:${uuidv4()}`
+}
+
+function generateBuildHandle(): string {
+  return uuidv4()
+}

packages/k8s/src/k8s/kaniko.ts

@@ -0,0 +1,95 @@
+import * as k8s from '@kubernetes/client-node'
+import * as path from 'path'
+import {
+  getRunnerPodName,
+  getVolumeClaimName,
+  MAX_POD_NAME_LENGTH,
+  RunnerInstanceLabel
+} from '../hooks/constants'
+import { POD_VOLUME_NAME } from '.'
+
+export const KANIKO_MOUNT_PATH = '/mnt/kaniko'
+
+function getKanikoName(): string {
+  return `${getRunnerPodName().substring(
+    0,
+    MAX_POD_NAME_LENGTH - '-kaniko'.length
+  )}-kaniko`
+}
+
+export function kanikoPod(
+  dockerfile: string,
+  destination: string,
+  secretName?: string
+): k8s.V1Pod {
+  const pod = new k8s.V1Pod()
+  pod.apiVersion = 'v1'
+  pod.kind = 'Pod'
+  pod.metadata = new k8s.V1ObjectMeta()
+  pod.metadata.name = getKanikoName()
+  const instanceLabel = new RunnerInstanceLabel()
+  pod.metadata.labels = {
+    [instanceLabel.key]: instanceLabel.value
+  }
+
+  const spec = new k8s.V1PodSpec()
+  const c = new k8s.V1Container()
+  c.image = 'gcr.io/kaniko-project/executor:latest'
+  c.name = 'kaniko'
+  c.imagePullPolicy = 'Always'
+  const prefix = (process.env.RUNNER_WORKSPACE as string).split('_work')[0]
+  const subPath = path
+    .dirname(dockerfile)
+    .substring(prefix.length + '_work/'.length)
+
+  c.volumeMounts = [
+    {
+      name: POD_VOLUME_NAME,
+      mountPath: KANIKO_MOUNT_PATH,
+      subPath,
+      readOnly: true
+    }
+  ]
+  c.args = [
+    `--dockerfile=${path.basename(dockerfile)}`,
+    `--context=dir://${KANIKO_MOUNT_PATH}`,
+    `--destination=${destination}`
+  ]
+  spec.containers = [c]
+  spec.dnsPolicy = 'ClusterFirst'
+  spec.restartPolicy = 'Never'
+  pod.spec = spec
+  const claimName: string = getVolumeClaimName()
+  pod.spec.volumes = [
+    {
+      name: POD_VOLUME_NAME,
+      persistentVolumeClaim: { claimName }
+    }
+  ]
+  if (secretName) {
+    const volumeName = 'docker-registry'
+    pod.spec.volumes.push({
+      name: volumeName,
+      projected: {
+        sources: [
+          {
+            secret: {
+              name: secretName,
+              items: [
+                {
+                  key: '.dockerconfigjson',
+                  path: 'config.json'
+                }
+              ]
+            }
+          }
+        ]
+      }
+    })
+    c.volumeMounts.push({
+      name: volumeName,
+      mountPath: '/kaniko/.docker/'
+    })
+  }
+  return pod
+}

packages/k8s/src/k8s/settings.ts

@@ -0,0 +1,73 @@
+import * as k8s from '@kubernetes/client-node'
+export const kc = new k8s.KubeConfig()
+
+kc.loadFromDefault()
+
+export const k8sApi = kc.makeApiClient(k8s.CoreV1Api)
+export const k8sBatchV1Api = kc.makeApiClient(k8s.BatchV1Api)
+export const k8sAuthorizationV1Api = kc.makeApiClient(k8s.AuthorizationV1Api)
+
+export const POD_VOLUME_NAME = 'work'
+export function namespace(): string {
+  if (process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']) {
+    return process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']
+  }
+
+  const context = kc.getContexts().find(ctx => ctx.namespace)
+  if (!context?.namespace) {
+    throw new Error(
+      'Failed to determine namespace, falling back to `default`. Namespace should be set in context, or in env variable "ACTIONS_RUNNER_KUBERNETES_NAMESPACE"'
+    )
+  }
+  return context.namespace
+}
+
+export function isLocalRegistrySet(): boolean {
+  const name = 'ACTIONS_RUNNER_CONTAINER_HOOKS_LOCAL_REGISTRY_HOST'
+  return !!process.env[name]
+}
+
+export function localRegistryHost(): string {
+  const name = 'ACTIONS_RUNNER_CONTAINER_HOOKS_LOCAL_REGISTRY_HOST'
+  if (process.env[name]) {
+    return process.env[name]
+  }
+  throw new Error(`environment variable ${name} is not set`)
+}
+
+export function localRegistryPort(): number {
+  const name = 'ACTIONS_RUNNER_CONTAINER_HOOKS_LOCAL_REGISTRY_PORT'
+  if (process.env[name]) {
+    return parseInt(process.env[name])
+  }
+  throw new Error(`environment variable ${name} is not set`)
+}
+
+export function localRegistryNodePort(): number {
+  const name = 'ACTIONS_RUNNER_CONTAINER_HOOKS_LOCAL_REGISTRY_NODE_PORT'
+  if (process.env[name]) {
+    return parseInt(process.env[name])
+  }
+  throw new Error(`environment variable ${name} is not set`)
+}
+
+export function remoteRegistryHost(): string {
+  const name = 'ACTIONS_RUNNER_CONTAINER_HOOKS_REMOTE_REGISTRY_HOST'
+  return process.env[name] || ''
+}
+
+export function remoteRegistryHandle(): string {
+  const name = 'ACTIONS_RUNNER_CONTAINER_HOOKS_REMOTE_REGISTRY_HANDLE'
+  if (process.env[name]) {
+    return process.env[name]
+  }
+  throw new Error(`environment variable ${name} is not set`)
+}
+
+export function remoteRegistrySecretName(): string {
+  const name = 'ACTIONS_RUNNER_CONTAINER_HOOKS_REMOTE_REGISTRY_SECRET_NAME'
+  if (process.env[name]) {
+    return process.env[name]
+  }
+  throw new Error(`environment variable ${name} is not set`)
+}

packages/k8s/tests/run-container-step-test.ts

@@ -3,11 +3,10 @@ import { TestHelper } from './test-setup'
 
 jest.useRealTimers()
 
-let testHelper: TestHelper
+describe('Run container step with image', () => {
+  let testHelper: TestHelper
+  let runContainerStepData: any
 
-let runContainerStepData: any
-
-describe('Run container step', () => {
   beforeEach(async () => {
     testHelper = new TestHelper()
     await testHelper.initialize()
@@ -39,3 +38,33 @@ describe('Run container step', () => {
     ).resolves.not.toThrow()
   })
 })
+
+describe('run container step with docker build', () => {
+  let testHelper: TestHelper
+  let runContainerStepData: any
+  beforeEach(async () => {
+    testHelper = new TestHelper()
+    await testHelper.initialize()
+    runContainerStepData = testHelper.getRunContainerStepDefinition()
+  })
+
+  afterEach(async () => {
+    await testHelper.cleanup()
+  })
+
+  it('should build container and execute docker action', async () => {
+    const { registryName, localRegistryPort, nodePort } =
+      await testHelper.createContainerRegistry()
+
+    process.env.ACTIONS_RUNNER_CONTAINER_HOOKS_LOCAL_REGISTRY_HOST =
+      registryName
+    process.env.ACTIONS_RUNNER_CONTAINER_HOOKS_LOCAL_REGISTRY_PORT =
+      localRegistryPort.toString()
+    process.env.ACTIONS_RUNNER_CONTAINER_HOOKS_LOCAL_REGISTRY_NODE_PORT =
+      nodePort.toString()
+    const actionPath = testHelper.initializeDockerAction()
+    const data = JSON.parse(JSON.stringify(runContainerStepData))
+    data.args.dockerfile = `${actionPath}/Dockerfile`
+    await expect(runContainerStep(data.args)).resolves.not.toThrow()
+  })
+})

packages/k8s/tests/run-script-step-test.ts

@@ -59,6 +59,7 @@ describe('Run script step', () => {
   it('should shold have env variables available', async () => {
     runScriptStepDefinition.args.entryPoint = 'bash'
 
+    runScriptStepDefinition.args.workingDirectory = '/' // set to '/' so that cd does not throw
     runScriptStepDefinition.args.entryPointArgs = [
       '-c',
       "'if [[ -z $NODE_ENV ]]; then exit 1; fi'"

packages/k8s/tests/test-setup.ts

@@ -2,7 +2,10 @@ import * as k8s from '@kubernetes/client-node'
 import * as fs from 'fs'
 import { HookData } from 'hooklib/lib'
 import * as path from 'path'
+import internal from 'stream'
 import { v4 as uuidv4 } from 'uuid'
+import { waitForPodPhases } from '../src/k8s'
+import { PodPhase } from '../src/k8s/utils'
 
 const kc = new k8s.KubeConfig()
 
@@ -10,6 +13,7 @@ kc.loadFromDefault()
 
 const k8sApi = kc.makeApiClient(k8s.CoreV1Api)
 const k8sStorageApi = kc.makeApiClient(k8s.StorageV1Api)
+const k8sAppsV1 = kc.makeApiClient(k8s.AppsV1Api)
 
 export class TestHelper {
   private tempDirPath: string
@@ -74,10 +78,19 @@ export class TestHelper {
         0
       )
       .catch(e => {})
+    await k8sApi
+      .deleteNamespacedPod(
+        `${this.podName}-kaniko`,
+        'default',
+        undefined,
+        undefined,
+        0
+      )
+      .catch(e => {})
   }
-  public createFile(fileName?: string): string {
+  public createFile(fileName?: string, content = ''): string {
     const filePath = `${this.tempDirPath}/${fileName || uuidv4()}`
-    fs.writeFileSync(filePath, '')
+    fs.writeFileSync(filePath, content)
     return filePath
   }
 
@@ -193,4 +206,237 @@ export class TestHelper {
     runContainerStep.args.registry = null
     return runContainerStep
   }
+
+  public async createContainerRegistry(): Promise<{
+    registryName: string
+    localRegistryPort: number
+    nodePort: number
+  }> {
+    const registryName = 'docker-registry'
+    const localRegistryPort = 5000
+    const nodePort = 31500
+
+    const cm = registryConfigMap(registryName, localRegistryPort)
+    const secret = registrySecret(registryName)
+    const ss = registryStatefulSet(registryName, localRegistryPort)
+    const svc = registryService(registryName, localRegistryPort, nodePort)
+    const namespace =
+      process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] || 'default'
+
+    await Promise.all([
+      k8sApi.createNamespacedConfigMap(namespace, cm),
+      k8sApi.createNamespacedSecret(namespace, secret)
+    ])
+    await k8sAppsV1.createNamespacedStatefulSet(namespace, ss)
+    await waitForPodPhases(
+      `${registryName}-0`,
+      new Set([PodPhase.RUNNING]),
+      new Set([PodPhase.PENDING])
+    )
+    await k8sApi.createNamespacedService(namespace, svc)
+    return {
+      registryName,
+      localRegistryPort,
+      nodePort
+    }
+  }
+
+  public initializeDockerAction(): string {
+    const actionPath = `${this.tempDirPath}/_work/_actions/example-handle/example-repo/example-branch/mock-directory`
+    fs.mkdirSync(actionPath, { recursive: true })
+    this.writeDockerfile(actionPath)
+    this.writeEntrypoint(actionPath)
+    return actionPath
+  }
+
+  private writeDockerfile(actionPath: string) {
+    const content = `FROM ubuntu:latest
+COPY entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]`
+    fs.writeFileSync(`${actionPath}/Dockerfile`, content)
+  }
+
+  private writeEntrypoint(actionPath) {
+    const content = `#!/bin/sh -l
+echo "Hello $1"
+time=$(date)
+echo "::set-output name=time::$time"`
+    const entryPointPath = `${actionPath}/entrypoint.sh`
+    fs.writeFileSync(entryPointPath, content)
+    fs.chmodSync(entryPointPath, 0o755)
+  }
+}
+
+function registryConfigMap(name: string, port: number): k8s.V1ConfigMap {
+  const REGISTRY_CONFIG_MAP_YAML = `
+storage:
+  filesystem:
+    rootdirectory: /var/lib/registry
+    maxthreads: 100
+health:
+  storagedriver:
+    enabled: true
+    interval: 10s
+    threshold: 3
+http:
+  addr: :${port}
+  headers:
+    X-Content-Type-Options:
+    - nosniff
+log:
+  fields:
+    service: registry
+storage:
+  cache:
+    blobdescriptor: inmemory
+version: 0.1
+`.trim()
+  const cm = new k8s.V1ConfigMap()
+  cm.apiVersion = 'v1'
+  cm.data = {
+    'config.yaml': REGISTRY_CONFIG_MAP_YAML
+  }
+  cm.kind = 'ConfigMap'
+  cm.metadata = new k8s.V1ObjectMeta()
+  cm.metadata.labels = { app: name }
+  cm.metadata.name = `${name}-config`
+
+  return cm
+}
+
+function registryStatefulSet(name: string, port: number): k8s.V1StatefulSet {
+  const ss = new k8s.V1StatefulSet()
+  ss.apiVersion = 'apps/v1'
+  ss.metadata = new k8s.V1ObjectMeta()
+  ss.metadata.name = name
+
+  const spec = new k8s.V1StatefulSetSpec()
+  spec.selector = new k8s.V1LabelSelector()
+  spec.selector.matchLabels = { app: 'docker-registry' }
+  spec.serviceName = 'registry'
+  spec.replicas = 1
+
+  const tmpl = new k8s.V1PodTemplateSpec()
+  tmpl.metadata = new k8s.V1ObjectMeta()
+  tmpl.metadata.labels = { app: name }
+  tmpl.spec = new k8s.V1PodSpec()
+  tmpl.spec.terminationGracePeriodSeconds = 5 // TODO: figure out for how long
+
+  const c = new k8s.V1Container()
+  c.command = ['/bin/registry', 'serve', '/etc/docker/registry/config.yaml']
+  c.env = [
+    {
+      name: 'REGISTRY_HTTP_SECRET',
+      valueFrom: {
+        secretKeyRef: {
+          key: 'haSharedSecret',
+          name: `${name}-secret`
+        }
+      }
+    },
+    {
+      name: 'REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY',
+      value: '/var/lib/registry'
+    }
+  ]
+  c.image = 'registry:2.6.2'
+  c.name = name
+  c.imagePullPolicy = 'IfNotPresent'
+  c.ports = [
+    {
+      containerPort: port,
+      protocol: 'TCP'
+    }
+  ]
+
+  c.volumeMounts = [
+    {
+      mountPath: '/etc/docker/registry',
+      name: 'docker-registry-config'
+    }
+  ]
+
+  c.livenessProbe = new k8s.V1Probe()
+  c.livenessProbe.failureThreshold = 3
+  c.livenessProbe.periodSeconds = 10
+  c.livenessProbe.successThreshold = 1
+  c.livenessProbe.timeoutSeconds = 1
+  c.livenessProbe.httpGet = new k8s.V1HTTPGetAction()
+  c.livenessProbe.httpGet.path = '/'
+  c.livenessProbe.httpGet.port = port
+  c.livenessProbe.httpGet.scheme = 'HTTP'
+
+  c.readinessProbe = new k8s.V1Probe()
+  c.readinessProbe.failureThreshold = 3
+  c.readinessProbe.periodSeconds = 10
+  c.readinessProbe.successThreshold = 1
+  c.readinessProbe.timeoutSeconds = 1
+  c.readinessProbe.httpGet = new k8s.V1HTTPGetAction()
+  c.readinessProbe.httpGet.path = '/'
+  c.readinessProbe.httpGet.port = port
+  c.readinessProbe.httpGet.scheme = 'HTTP'
+
+  tmpl.spec.containers = [c]
+  tmpl.spec.volumes = [
+    {
+      name: `${name}-config`,
+      configMap: {
+        name: `${name}-config`
+      }
+    }
+  ]
+
+  spec.template = tmpl
+  ss.spec = spec
+
+  return ss
+}
+function registryService(
+  name: string,
+  port: number,
+  nodePort: number
+): k8s.V1Service {
+  const svc = new k8s.V1Service()
+  svc.apiVersion = 'v1'
+  svc.kind = 'Service'
+  svc.metadata = new k8s.V1ObjectMeta()
+  svc.metadata.name = name
+  svc.metadata.labels = {
+    app: name
+  }
+  const spec = new k8s.V1ServiceSpec()
+  spec.externalTrafficPolicy = 'Cluster'
+  spec.ports = [
+    {
+      name: 'registry',
+      nodePort: nodePort,
+      port: port,
+      protocol: 'TCP',
+      targetPort: port
+    }
+  ]
+  spec.selector = {
+    app: name
+  }
+  spec.sessionAffinity = 'None'
+  spec.type = 'NodePort'
+  svc.spec = spec
+
+  return svc
+}
+
+function registrySecret(name: string): k8s.V1Secret {
+  const secret = new k8s.V1Secret()
+  secret.apiVersion = 'v1'
+  secret.data = { haSharedSecret: 'U29tZVZlcnlTdHJpbmdTZWNyZXQK' }
+  secret.kind = 'Secret'
+  secret.metadata = new k8s.V1ObjectMeta()
+  secret.metadata.labels = {
+    app: name,
+    chart: `${name}-1.4.3`
+  }
+  secret.metadata.name = `${name}-secret`
+  secret.type = 'Opaque'
+
+  return secret
 }