Implement yaml extensions overwriting the default pod/container spec (#75)

* Implement yaml extensions overwriting the default pod/container spec

* format files

* Extend specs for container job and include docker and k8s tests in k8s

* Create table tests for docker tests

* included warnings and extracted append logic as generic

* updated merge to allow for file read

* reverted back examples and k8s/tests

* reverted back docker tests

* Tests for extension prepare-job

* Fix lint and format and merge error

* Added basic test for container step

* revert hooklib since new definition for container options is received from a file

* revert docker options since create options are a string

* Fix revert

* Update package locks and deps

* included example of extension.yaml. Added side-car container that was missing

* Ignore spec modification for the service containers, change selector to

* fix lint error

* Add missing image override

* Add comment explaining merge object meta with job and pod

* fix test

packages/k8s/src/hooks/constants.ts

@@ -42,6 +42,7 @@ export function getSecretName(): string {
 export const MAX_POD_NAME_LENGTH = 63
 export const STEP_POD_NAME_SUFFIX_LENGTH = 8
 export const JOB_CONTAINER_NAME = 'job'
+export const JOB_CONTAINER_EXTENSION_NAME = '$job'
 
 export class RunnerInstanceLabel {
   private podName: string

packages/k8s/src/hooks/prepare-job.ts

@@ -1,7 +1,12 @@
 import * as core from '@actions/core'
 import * as io from '@actions/io'
 import * as k8s from '@kubernetes/client-node'
-import { ContextPorts, prepareJobArgs, writeToResponseFile } from 'hooklib'
+import {
+  JobContainerInfo,
+  ContextPorts,
+  PrepareJobArgs,
+  writeToResponseFile
+} from 'hooklib'
 import path from 'path'
 import {
   containerPorts,
@@ -15,12 +20,14 @@ import {
   DEFAULT_CONTAINER_ENTRY_POINT,
   DEFAULT_CONTAINER_ENTRY_POINT_ARGS,
   generateContainerName,
+  mergeContainerWithOptions,
+  readExtensionFromFile,
   PodPhase
 } from '../k8s/utils'
-import { JOB_CONTAINER_NAME } from './constants'
+import { JOB_CONTAINER_EXTENSION_NAME, JOB_CONTAINER_NAME } from './constants'
 
 export async function prepareJob(
-  args: prepareJobArgs,
+  args: PrepareJobArgs,
   responseFile
 ): Promise<void> {
   if (!args.container) {
@@ -28,26 +35,46 @@ export async function prepareJob(
   }
 
   await prunePods()
+
+  const extension = readExtensionFromFile()
   await copyExternalsToRoot()
+
   let container: k8s.V1Container | undefined = undefined
   if (args.container?.image) {
     core.debug(`Using image '${args.container.image}' for job image`)
-    container = createContainerSpec(args.container, JOB_CONTAINER_NAME, true)
+    container = createContainerSpec(
+      args.container,
+      JOB_CONTAINER_NAME,
+      true,
+      extension
+    )
   }
 
   let services: k8s.V1Container[] = []
   if (args.services?.length) {
     services = args.services.map(service => {
       core.debug(`Adding service '${service.image}' to pod definition`)
-      return createContainerSpec(service, generateContainerName(service.image))
+      return createContainerSpec(
+        service,
+        generateContainerName(service.image),
+        false,
+        undefined
+      )
     })
   }
+
   if (!container && !services?.length) {
     throw new Error('No containers exist, skipping hook invocation')
   }
+
   let createdPod: k8s.V1Pod | undefined = undefined
   try {
-    createdPod = await createPod(container, services, args.container.registry)
+    createdPod = await createPod(
+      container,
+      services,
+      args.container.registry,
+      extension
+    )
   } catch (err) {
     await prunePods()
     throw new Error(`failed to create job pod: ${err}`)
@@ -153,9 +180,10 @@ async function copyExternalsToRoot(): Promise<void> {
 }
 
 export function createContainerSpec(
-  container,
+  container: JobContainerInfo,
   name: string,
-  jobContainer = false
+  jobContainer = false,
+  extension?: k8s.V1PodTemplateSpec
 ): k8s.V1Container {
   if (!container.entryPoint && jobContainer) {
     container.entryPoint = DEFAULT_CONTAINER_ENTRY_POINT
@@ -193,5 +221,17 @@ export function createContainerSpec(
     jobContainer
   )
 
+  if (!extension) {
+    return podContainer
+  }
+
+  const from = extension.spec?.containers?.find(
+    c => c.name === JOB_CONTAINER_EXTENSION_NAME
+  )
+
+  if (from) {
+    mergeContainerWithOptions(podContainer, from)
+  }
+
   return podContainer
 }

packages/k8s/src/hooks/run-container-step.ts

@@ -10,8 +10,13 @@ import {
   waitForJobToComplete,
   waitForPodPhases
 } from '../k8s'
-import { containerVolumes, PodPhase } from '../k8s/utils'
-import { JOB_CONTAINER_NAME } from './constants'
+import {
+  containerVolumes,
+  PodPhase,
+  mergeContainerWithOptions,
+  readExtensionFromFile
+} from '../k8s/utils'
+import { JOB_CONTAINER_EXTENSION_NAME, JOB_CONTAINER_NAME } from './constants'
 
 export async function runContainerStep(
   stepContainer: RunContainerStepArgs
@@ -25,10 +30,12 @@ export async function runContainerStep(
     secretName = await createSecretForEnvs(stepContainer.environmentVariables)
   }
 
-  core.debug(`Created secret ${secretName} for container job envs`)
-  const container = createPodSpec(stepContainer, secretName)
+  const extension = readExtensionFromFile()
 
-  const job = await createJob(container)
+  core.debug(`Created secret ${secretName} for container job envs`)
+  const container = createContainerSpec(stepContainer, secretName, extension)
+
+  const job = await createJob(container, extension)
   if (!job.metadata?.name) {
     throw new Error(
       `Expected job ${JSON.stringify(
@@ -69,9 +76,10 @@ export async function runContainerStep(
   return Number(exitCode) || 1
 }
 
-function createPodSpec(
+function createContainerSpec(
   container: RunContainerStepArgs,
-  secretName?: string
+  secretName?: string,
+  extension?: k8s.V1PodTemplateSpec
 ): k8s.V1Container {
   const podContainer = new k8s.V1Container()
   podContainer.name = JOB_CONTAINER_NAME
@@ -96,5 +104,16 @@ function createPodSpec(
   }
   podContainer.volumeMounts = containerVolumes(undefined, false, true)
 
+  if (!extension) {
+    return podContainer
+  }
+
+  const from = extension.spec?.containers?.find(
+    c => c.name === JOB_CONTAINER_EXTENSION_NAME
+  )
+  if (from) {
+    mergeContainerWithOptions(podContainer, from)
+  }
+
   return podContainer
 }

packages/k8s/src/k8s/index.ts

@@ -10,7 +10,7 @@ import {
   getVolumeClaimName,
   RunnerInstanceLabel
 } from '../hooks/constants'
-import { PodPhase } from './utils'
+import { PodPhase, mergePodSpecWithOptions, mergeObjectMeta } from './utils'
 
 const kc = new k8s.KubeConfig()
 
@@ -58,7 +58,8 @@ export const requiredPermissions = [
 export async function createPod(
   jobContainer?: k8s.V1Container,
   services?: k8s.V1Container[],
-  registry?: Registry
+  registry?: Registry,
+  extension?: k8s.V1PodTemplateSpec
 ): Promise<k8s.V1Pod> {
   const containers: k8s.V1Container[] = []
   if (jobContainer) {
@@ -80,6 +81,7 @@ export async function createPod(
   appPod.metadata.labels = {
     [instanceLabel.key]: instanceLabel.value
   }
+  appPod.metadata.annotations = {}
 
   appPod.spec = new k8s.V1PodSpec()
   appPod.spec.containers = containers
@@ -103,12 +105,21 @@ export async function createPod(
     appPod.spec.imagePullSecrets = [secretReference]
   }
 
+  if (extension?.metadata) {
+    mergeObjectMeta(appPod, extension.metadata)
+  }
+
+  if (extension?.spec) {
+    mergePodSpecWithOptions(appPod.spec, extension.spec)
+  }
+
   const { body } = await k8sApi.createNamespacedPod(namespace(), appPod)
   return body
 }
 
 export async function createJob(
-  container: k8s.V1Container
+  container: k8s.V1Container,
+  extension?: k8s.V1PodTemplateSpec
 ): Promise<k8s.V1Job> {
   const runnerInstanceLabel = new RunnerInstanceLabel()
 
@@ -118,6 +129,7 @@ export async function createJob(
   job.metadata = new k8s.V1ObjectMeta()
   job.metadata.name = getStepPodName()
   job.metadata.labels = { [runnerInstanceLabel.key]: runnerInstanceLabel.value }
+  job.metadata.annotations = {}
 
   job.spec = new k8s.V1JobSpec()
   job.spec.ttlSecondsAfterFinished = 300
@@ -125,6 +137,9 @@ export async function createJob(
   job.spec.template = new k8s.V1PodTemplateSpec()
 
   job.spec.template.spec = new k8s.V1PodSpec()
+  job.spec.template.metadata = new k8s.V1ObjectMeta()
+  job.spec.template.metadata.labels = {}
+  job.spec.template.metadata.annotations = {}
   job.spec.template.spec.containers = [container]
   job.spec.template.spec.restartPolicy = 'Never'
   job.spec.template.spec.nodeName = await getCurrentNodeName()
@@ -137,6 +152,17 @@ export async function createJob(
     }
   ]
 
+  if (extension) {
+    if (extension.metadata) {
+      // apply metadata both to the job and the pod created by the job
+      mergeObjectMeta(job, extension.metadata)
+      mergeObjectMeta(job.spec.template, extension.metadata)
+    }
+    if (extension.spec) {
+      mergePodSpecWithOptions(job.spec.template.spec, extension.spec)
+    }
+  }
+
   const { body } = await k8sBatchV1Api.createNamespacedJob(namespace(), job)
   return body
 }
@@ -555,3 +581,8 @@ export function containerPorts(
   }
   return ports
 }
+
+export async function getPodByName(name): Promise<k8s.V1Pod> {
+  const { body } = await k8sApi.readNamespacedPod(name, namespace())
+  return body
+}

packages/k8s/src/k8s/utils.ts

@@ -1,5 +1,7 @@
 import * as k8s from '@kubernetes/client-node'
 import * as fs from 'fs'
+import * as yaml from 'js-yaml'
+import * as core from '@actions/core'
 import { Mount } from 'hooklib'
 import * as path from 'path'
 import { v1 as uuidv4 } from 'uuid'
@@ -8,6 +10,8 @@ import { POD_VOLUME_NAME } from './index'
 export const DEFAULT_CONTAINER_ENTRY_POINT_ARGS = [`-f`, `/dev/null`]
 export const DEFAULT_CONTAINER_ENTRY_POINT = 'tail'
 
+export const ENV_HOOK_TEMPLATE_PATH = 'ACTIONS_RUNNER_CONTAINER_HOOK_TEMPLATE'
+
 export function containerVolumes(
   userMountVolumes: Mount[] = [],
   jobContainer = true,
@@ -159,6 +163,100 @@ export function generateContainerName(image: string): string {
   return name
 }
 
+// Overwrite or append based on container options
+//
+// Keep in mind, envs and volumes could be passed as fields in container definition
+// so default volume mounts and envs are appended first, and then create options are used
+// to append more values
+//
+// Rest of the fields are just applied
+// For example, container.createOptions.container.image is going to overwrite container.image field
+export function mergeContainerWithOptions(
+  base: k8s.V1Container,
+  from: k8s.V1Container
+): void {
+  for (const [key, value] of Object.entries(from)) {
+    if (key === 'name') {
+      core.warning("Skipping name override: name can't be overwritten")
+      continue
+    } else if (key === 'image') {
+      core.warning("Skipping image override: image can't be overwritten")
+      continue
+    } else if (key === 'env') {
+      const envs = value as k8s.V1EnvVar[]
+      base.env = mergeLists(base.env, envs)
+    } else if (key === 'volumeMounts' && value) {
+      const volumeMounts = value as k8s.V1VolumeMount[]
+      base.volumeMounts = mergeLists(base.volumeMounts, volumeMounts)
+    } else if (key === 'ports' && value) {
+      const ports = value as k8s.V1ContainerPort[]
+      base.ports = mergeLists(base.ports, ports)
+    } else {
+      base[key] = value
+    }
+  }
+}
+
+export function mergePodSpecWithOptions(
+  base: k8s.V1PodSpec,
+  from: k8s.V1PodSpec
+): void {
+  for (const [key, value] of Object.entries(from)) {
+    if (key === 'containers') {
+      base.containers.push(
+        ...from.containers.filter(e => !e.name?.startsWith('$'))
+      )
+    } else if (key === 'volumes' && value) {
+      const volumes = value as k8s.V1Volume[]
+      base.volumes = mergeLists(base.volumes, volumes)
+    } else {
+      base[key] = value
+    }
+  }
+}
+
+export function mergeObjectMeta(
+  base: { metadata?: k8s.V1ObjectMeta },
+  from: k8s.V1ObjectMeta
+): void {
+  if (!base.metadata?.labels || !base.metadata?.annotations) {
+    throw new Error(
+      "Can't merge metadata: base.metadata or base.annotations field is undefined"
+    )
+  }
+  if (from?.labels) {
+    for (const [key, value] of Object.entries(from.labels)) {
+      if (base.metadata?.labels?.[key]) {
+        core.warning(`Label ${key} is already defined and will be overwritten`)
+      }
+      base.metadata.labels[key] = value
+    }
+  }
+
+  if (from?.annotations) {
+    for (const [key, value] of Object.entries(from.annotations)) {
+      if (base.metadata?.annotations?.[key]) {
+        core.warning(
+          `Annotation ${key} is already defined and will be overwritten`
+        )
+      }
+      base.metadata.annotations[key] = value
+    }
+  }
+}
+
+export function readExtensionFromFile(): k8s.V1PodTemplateSpec | undefined {
+  const filePath = process.env[ENV_HOOK_TEMPLATE_PATH]
+  if (!filePath) {
+    return undefined
+  }
+  const doc = yaml.load(fs.readFileSync(filePath, 'utf8'))
+  if (!doc || typeof doc !== 'object') {
+    throw new Error(`Failed to parse ${filePath}`)
+  }
+  return doc as k8s.V1PodTemplateSpec
+}
+
 export enum PodPhase {
   PENDING = 'Pending',
   RUNNING = 'Running',
@@ -167,3 +265,12 @@ export enum PodPhase {
   UNKNOWN = 'Unknown',
   COMPLETED = 'Completed'
 }
+
+function mergeLists<T>(base?: T[], from?: T[]): T[] {
+  const b: T[] = base || []
+  if (!from?.length) {
+    return b
+  }
+  b.push(...from)
+  return b
+}