Initial Commit

This commit is contained in:
Thomas Boop
2022-06-02 15:53:11 -04:00
parent 4c8cc497b3
commit 6159767f90
70 changed files with 30723 additions and 0 deletions

View File

@@ -0,0 +1,9 @@
export function getRunnerLabel(): string {
const name = process.env.RUNNER_NAME
if (!name) {
throw new Error(
"'RUNNER_NAME' env is required, please contact your self hosted runner administrator"
)
}
return Buffer.from(name).toString('hex')
}

View File

@@ -0,0 +1,413 @@
import * as core from '@actions/core'
import * as fs from 'fs'
import {
ContainerInfo,
JobContainerInfo,
RunContainerStepArgs,
ServiceContainerInfo,
StepContainerInfo
} from 'hooklib/lib'
import path from 'path'
import { env } from 'process'
import { v4 as uuidv4 } from 'uuid'
import { runDockerCommand, RunDockerCommandOptions } from '../utils'
import { getRunnerLabel } from './constants'
export async function createContainer(
args: ContainerInfo,
name: string,
network: string
): Promise<ContainerMetadata> {
if (!args.image) {
throw new Error('Image was expected')
}
const dockerArgs: string[] = ['create']
dockerArgs.push(`--label=${getRunnerLabel()}`)
dockerArgs.push(`--network=${network}`)
if ((args as ServiceContainerInfo)?.contextName) {
dockerArgs.push(
`--network-alias=${(args as ServiceContainerInfo)?.contextName}`
)
}
dockerArgs.push('--name', name)
if (args?.portMappings?.length) {
for (const portMapping of args.portMappings) {
dockerArgs.push('-p', portMapping)
}
}
if (args.createOptions) {
dockerArgs.push(...args.createOptions.split(' '))
}
if (args.environmentVariables) {
for (const [key, value] of Object.entries(args.environmentVariables)) {
dockerArgs.push('-e')
if (!value) {
dockerArgs.push(`"${key}"`)
} else {
dockerArgs.push(`"${key}=${value}"`)
}
}
}
const mountVolumes = [
...(args.userMountVolumes || []),
...((args as JobContainerInfo | StepContainerInfo).systemMountVolumes || [])
]
for (const mountVolume of mountVolumes) {
dockerArgs.push(
`-v=${mountVolume.sourceVolumePath}:${mountVolume.targetVolumePath}`
)
}
if (args.entryPoint) {
dockerArgs.push(`--entrypoint`)
dockerArgs.push(args.entryPoint)
}
dockerArgs.push(args.image)
if (args.entryPointArgs) {
for (const entryPointArg of args.entryPointArgs) {
dockerArgs.push(entryPointArg)
}
}
const id = (await runDockerCommand(dockerArgs)).trim()
if (!id) {
throw new Error('Could not read id from docker command')
}
const response: ContainerMetadata = { id, image: args.image }
if (network) {
response.network = network
}
response.ports = []
if ((args as ServiceContainerInfo).contextName) {
response['contextName'] = (args as ServiceContainerInfo).contextName
}
return response
}
export async function containerPull(
image: string,
configLocation: string
): Promise<void> {
const dockerArgs: string[] = ['pull']
if (configLocation) {
dockerArgs.push('--config')
dockerArgs.push(configLocation)
}
dockerArgs.push(image)
for (let i = 0; i < 3; i++) {
try {
await runDockerCommand(dockerArgs)
return
} catch {
core.info(`docker pull failed on attempt: ${i + 1}`)
}
}
throw new Error('Exiting docker pull after 3 failed attempts')
}
export async function containerStart(id: string): Promise<void> {
const dockerArgs: string[] = ['start']
dockerArgs.push(`${id}`)
await runDockerCommand(dockerArgs)
}
export async function containerStop(id: string | string[]): Promise<void> {
const dockerArgs: string[] = ['stop']
if (Array.isArray(id)) {
for (const v of id) {
dockerArgs.push(v)
}
} else {
dockerArgs.push(id)
}
await runDockerCommand(dockerArgs)
}
export async function containerRemove(id: string | string[]): Promise<void> {
const dockerArgs: string[] = ['rm']
dockerArgs.push('--force')
if (Array.isArray(id)) {
for (const v of id) {
dockerArgs.push(v)
}
} else {
dockerArgs.push(id)
}
await runDockerCommand(dockerArgs)
}
export async function containerBuild(
args: RunContainerStepArgs,
tag: string
): Promise<void> {
const context = path.dirname(`${env.GITHUB_WORKSPACE}/${args.dockerfile}`)
const dockerArgs: string[] = ['build']
dockerArgs.push('-t', tag)
dockerArgs.push('-f', `${env.GITHUB_WORKSPACE}/${args.dockerfile}`)
dockerArgs.push(context)
// TODO: figure out build working directory
await runDockerCommand(dockerArgs, {
workingDir: args['buildWorkingDirectory']
})
}
export async function containerLogs(id: string): Promise<void> {
const dockerArgs: string[] = ['logs']
dockerArgs.push('--details')
dockerArgs.push(id)
await runDockerCommand(dockerArgs)
}
export async function containerNetworkRemove(network: string): Promise<void> {
const dockerArgs: string[] = ['network']
dockerArgs.push('rm')
dockerArgs.push(network)
await runDockerCommand(dockerArgs)
}
export async function containerPrune(): Promise<void> {
const dockerPSArgs: string[] = [
'ps',
'--all',
'--quiet',
'--no-trunc',
'--filter',
`label=${getRunnerLabel()}`
]
const res = (await runDockerCommand(dockerPSArgs)).trim()
if (res) {
await containerRemove(res.split('\n'))
}
}
async function containerHealthStatus(id: string): Promise<ContainerHealth> {
const dockerArgs = [
'inspect',
'--format="{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}"',
id
]
const result = (await runDockerCommand(dockerArgs)).trim().replace(/"/g, '')
if (
result === ContainerHealth.Healthy ||
result === ContainerHealth.Starting ||
result === ContainerHealth.Unhealthy
) {
return result
}
return ContainerHealth.None
}
export async function healthCheck({
id,
image
}: ContainerMetadata): Promise<void> {
let health = await containerHealthStatus(id)
if (health === ContainerHealth.None) {
core.info(
`Healthcheck is not set for container ${image}, considered as ${ContainerHealth.Healthy}`
)
return
}
let tries = 1
while (health === ContainerHealth.Starting && tries < 13) {
const backOffSeconds = Math.pow(2, tries)
core.info(
`Container '${image}' is '${health}', retry in ${backOffSeconds} seconds`
)
await new Promise(resolve => setTimeout(resolve, 1000 * backOffSeconds))
tries++
health = await containerHealthStatus(id)
}
if (health !== ContainerHealth.Healthy) {
throw new String(
`Container '${image}' is unhealthy with status '${health}'`
)
}
}
export async function containerPorts(id: string): Promise<string[]> {
const dockerArgs = ['port', id]
const portMappings = (await runDockerCommand(dockerArgs)).trim()
return portMappings.split('\n')
}
export async function registryLogin(args): Promise<string> {
if (!args.registry) {
return ''
}
const credentials = {
username: args.registry.username,
password: args.registry.password
}
const configLocation = `${env.RUNNER_TEMP}/.docker_${uuidv4()}`
fs.mkdirSync(configLocation)
try {
await dockerLogin(configLocation, args.registry.serverUrl, credentials)
} catch (error) {
fs.rmdirSync(configLocation, { recursive: true })
throw error
}
return configLocation
}
export async function registryLogout(configLocation: string): Promise<void> {
if (configLocation) {
await dockerLogout(configLocation)
fs.rmdirSync(configLocation, { recursive: true })
}
}
async function dockerLogin(
configLocation: string,
registry: string,
credentials: { username: string; password: string }
): Promise<void> {
const credentialsArgs =
credentials.username && credentials.password
? ['-u', credentials.username, '--password-stdin']
: []
const dockerArgs = [
'--config',
configLocation,
'login',
...credentialsArgs,
registry
]
const options: RunDockerCommandOptions =
credentials.username && credentials.password
? {
input: Buffer.from(credentials.password, 'utf-8')
}
: {}
await runDockerCommand(dockerArgs, options)
}
async function dockerLogout(configLocation: string): Promise<void> {
const dockerArgs = ['--config', configLocation, 'logout']
await runDockerCommand(dockerArgs)
}
export async function containerExecStep(
args,
containerId: string
): Promise<void> {
const dockerArgs: string[] = ['exec', '-i']
dockerArgs.push(`--workdir=${args.workingDirectory}`)
for (const [key, value] of Object.entries(args['environmentVariables'])) {
dockerArgs.push('-e')
if (!value) {
dockerArgs.push(`"${key}"`)
} else {
dockerArgs.push(`"${key}=${value}"`)
}
}
// Todo figure out prepend path and update it here
// (we need to pass path in as -e Path={fullpath}) where {fullpath is the prepend path added to the current containers path}
dockerArgs.push(containerId)
dockerArgs.push(args.entryPoint)
for (const entryPointArg of args.entryPointArgs) {
dockerArgs.push(entryPointArg)
}
await runDockerCommand(dockerArgs)
}
export async function containerRun(
args: RunContainerStepArgs,
name: string,
network: string
): Promise<void> {
if (!args.image) {
throw new Error('expected image to be set')
}
const dockerArgs: string[] = ['run', '--rm']
dockerArgs.push('--name', name)
dockerArgs.push(`--workdir=${args.workingDirectory}`)
dockerArgs.push(`--label=${getRunnerLabel()}`)
dockerArgs.push(`--network=${network}`)
if (args.createOptions) {
dockerArgs.push(...args.createOptions.split(' '))
}
if (args.environmentVariables) {
for (const [key, value] of Object.entries(args.environmentVariables)) {
// Pass in this way to avoid printing secrets
env[key] = value ?? undefined
dockerArgs.push('-e')
dockerArgs.push(key)
}
}
const mountVolumes = [
...(args.userMountVolumes || []),
...(args.systemMountVolumes || [])
]
for (const mountVolume of mountVolumes) {
dockerArgs.push(`-v`)
dockerArgs.push(
`${mountVolume.sourceVolumePath}:${mountVolume.targetVolumePath}${
mountVolume.readOnly ? ':ro' : ''
}`
)
}
if (args['entryPoint']) {
dockerArgs.push(`--entrypoint`)
dockerArgs.push(args['entryPoint'])
}
dockerArgs.push(args.image)
if (args.entryPointArgs) {
for (const entryPointArg of args.entryPointArgs) {
dockerArgs.push(entryPointArg)
}
}
await runDockerCommand(dockerArgs)
}
export async function isContainerAlpine(containerId: string): Promise<boolean> {
const dockerArgs: string[] = [
'exec',
containerId,
'sh',
'-c',
"[ $(cat /etc/*release* | grep -i -e '^ID=*alpine*' -c) != 0 ] || exit 1"
]
try {
await runDockerCommand(dockerArgs)
return true
} catch {
return false
}
}
enum ContainerHealth {
Starting = 'starting',
Healthy = 'healthy',
Unhealthy = 'unhealthy',
None = 'none'
}
export interface ContainerMetadata {
id: string
image: string
network?: string
ports?: string[]
contextName?: string
}

View File

@@ -0,0 +1,2 @@
export * from './container'
export * from './network'

View File

@@ -0,0 +1,26 @@
import { runDockerCommand } from '../utils'
import { getRunnerLabel } from './constants'
export async function networkCreate(networkName): Promise<void> {
const dockerArgs: string[] = ['network', 'create']
dockerArgs.push('--label')
dockerArgs.push(getRunnerLabel())
dockerArgs.push(networkName)
await runDockerCommand(dockerArgs)
}
export async function networkRemove(networkName): Promise<void> {
const dockerArgs: string[] = ['network']
dockerArgs.push('rm')
dockerArgs.push(networkName)
await runDockerCommand(dockerArgs)
}
export async function networkPrune(): Promise<void> {
const dockerArgs: string[] = ['network']
dockerArgs.push('prune')
dockerArgs.push('--force')
dockerArgs.push(`--filter`)
dockerArgs.push(`label=${getRunnerLabel()}`)
await runDockerCommand(dockerArgs)
}

View File

@@ -0,0 +1,21 @@
import {
containerRemove,
containerNetworkRemove
} from '../dockerCommands/container'
// eslint-disable-next-line @typescript-eslint/no-unused-vars
export async function cleanupJob(args, state, responseFile): Promise<void> {
const containerIds: string[] = []
if (state?.container) {
containerIds.push(state.container)
}
if (state?.services) {
containerIds.push(state.services)
}
if (containerIds.length > 0) {
await containerRemove(containerIds)
}
if (state.network) {
await containerNetworkRemove(state.network)
}
}

View File

@@ -0,0 +1,4 @@
export * from './cleanup-job'
export * from './prepare-job'
export * from './run-script-step'
export * from './run-container-step'

View File

@@ -0,0 +1,205 @@
import * as core from '@actions/core'
import { ContextPorts, PrepareJobArgs, writeToResponseFile } from 'hooklib/lib'
import { exit } from 'process'
import { v4 as uuidv4 } from 'uuid'
import {
ContainerMetadata,
containerPorts,
containerPrune,
containerPull,
containerStart,
createContainer,
healthCheck,
isContainerAlpine,
registryLogin,
registryLogout
} from '../dockerCommands/container'
import { networkCreate, networkPrune } from '../dockerCommands/network'
import { sanitize } from '../utils'
export async function prepareJob(
args: PrepareJobArgs,
responseFile
): Promise<void> {
await containerPrune()
await networkPrune()
const container = args.container
const services = args.services
if (!container?.image && !services?.length) {
core.info('No containers exist, skipping hook invocation')
exit(0)
}
const networkName = generateNetworkName()
// Create network
await networkCreate(networkName)
// Create Job Container
let containerMetadata: ContainerMetadata | undefined = undefined
if (!container?.image) {
core.info('No job container provided, skipping')
} else {
setupContainer(container)
const configLocation = await registryLogin(container.registry)
try {
await containerPull(container.image, configLocation)
} finally {
await registryLogout(configLocation)
}
containerMetadata = await createContainer(
container,
generateContainerName(container.image),
networkName
)
if (!containerMetadata?.id) {
throw new Error('Failed to create container')
}
await containerStart(containerMetadata?.id)
}
// Create Service Containers
const servicesMetadata: ContainerMetadata[] = []
if (!services?.length) {
core.info('No service containers provided, skipping')
} else {
for (const service of services) {
const configLocation = await registryLogin(service.registry)
try {
await containerPull(service.image, configLocation)
} finally {
await registryLogout(configLocation)
}
setupContainer(service)
const response = await createContainer(
service,
generateContainerName(service.image),
networkName
)
servicesMetadata.push(response)
await containerStart(response.id)
}
}
if (
(container && !containerMetadata?.id) ||
(services?.length && servicesMetadata.some(s => !s.id))
) {
throw new Error(
`Not all containers are started correctly ${
containerMetadata?.id
}, ${servicesMetadata.map(e => e.id).join(',')}`
)
}
const isAlpine = await isContainerAlpine(containerMetadata!.id)
if (containerMetadata?.id) {
containerMetadata.ports = await containerPorts(containerMetadata.id)
}
if (servicesMetadata?.length) {
for (const serviceMetadata of servicesMetadata) {
serviceMetadata.ports = await containerPorts(serviceMetadata.id)
}
}
const healthChecks: Promise<void>[] = [healthCheck(containerMetadata!)]
for (const service of servicesMetadata) {
healthChecks.push(healthCheck(service))
}
try {
await Promise.all(healthChecks)
core.info('All services are healthy')
} catch (error) {
core.error(`Failed to initialize containers, ${error}`)
throw new Error(`Failed to initialize containers, ${error}`)
}
generateResponseFile(
responseFile,
networkName,
containerMetadata,
servicesMetadata,
isAlpine
)
}
function generateResponseFile(
responseFile: string,
networkName: string,
containerMetadata?: ContainerMetadata,
servicesMetadata?: ContainerMetadata[],
isAlpine = false
): void {
// todo figure out if we are alpine
const response = {
state: { network: networkName },
context: {},
isAlpine
}
if (containerMetadata) {
response.state['container'] = containerMetadata.id
const contextMeta = JSON.parse(JSON.stringify(containerMetadata))
if (containerMetadata.ports) {
contextMeta.ports = transformDockerPortsToContextPorts(containerMetadata)
}
response.context['container'] = contextMeta
if (containerMetadata.ports) {
response.context['container'].ports =
transformDockerPortsToContextPorts(containerMetadata)
}
}
if (servicesMetadata && servicesMetadata.length > 0) {
response.state['services'] = []
response.context['services'] = []
for (const meta of servicesMetadata) {
response.state['services'].push(meta.id)
const contextMeta = JSON.parse(JSON.stringify(meta))
if (contextMeta.ports) {
contextMeta.ports = transformDockerPortsToContextPorts(contextMeta)
}
response.context['services'].push(contextMeta)
}
}
writeToResponseFile(responseFile, JSON.stringify(response))
}
function setupContainer(container): void {
container.entryPointArgs = [`-f`, `/dev/null`]
container.entryPoint = 'tail'
}
function generateNetworkName(): string {
return `github_network_${uuidv4()}`
}
function generateContainerName(container): string {
const randomAlias = uuidv4().replace(/-/g, '')
const randomSuffix = uuidv4().substring(0, 6)
return `${randomAlias}_${sanitize(container.image)}_${randomSuffix}`
}
function transformDockerPortsToContextPorts(
meta: ContainerMetadata
): ContextPorts {
// ex: '80/tcp -> 0.0.0.0:80'
const re = /^(\d+)\/(\w+)? -> (.*):(\d+)$/
const contextPorts: ContextPorts = {}
if (meta.ports) {
for (const port of meta.ports) {
const matches = port.match(re)
if (!matches) {
throw new Error(
'Container ports could not match the regex: "^(\\d+)\\/(\\w+)? -> (.*):(\\d+)$"'
)
}
contextPorts[matches[1]] = matches[matches.length - 1]
}
}
return contextPorts
}

View File

@@ -0,0 +1,39 @@
import {
containerBuild,
registryLogin,
registryLogout,
containerPull,
containerRun
} from '../dockerCommands'
import { v4 as uuidv4 } from 'uuid'
import * as core from '@actions/core'
import { RunContainerStepArgs } from 'hooklib/lib'
import { getRunnerLabel } from '../dockerCommands/constants'
export async function runContainerStep(
args: RunContainerStepArgs,
state
): Promise<void> {
const tag = generateBuildTag() // for docker build
if (!args.image) {
core.error('expected an image')
} else {
if (args.dockerfile) {
await containerBuild(args, tag)
args.image = tag
} else {
const configLocation = await registryLogin(args)
try {
await containerPull(args.image, configLocation)
} finally {
await registryLogout(configLocation)
}
}
}
// container will get pruned at the end of the job based on the label, no need to cleanup here
await containerRun(args, tag.split(':')[1], state.network)
}
function generateBuildTag(): string {
return `${getRunnerLabel()}:${uuidv4().substring(0, 6)}`
}

View File

@@ -0,0 +1,9 @@
import { RunScriptStepArgs } from 'hooklib/lib'
import { containerExecStep } from '../dockerCommands'
export async function runScriptStep(
args: RunScriptStepArgs,
state
): Promise<void> {
await containerExecStep(args, state.container)
}

View File

@@ -0,0 +1,48 @@
import * as core from '@actions/core'
import {
Command,
getInputFromStdin,
PrepareJobArgs,
RunContainerStepArgs,
RunScriptStepArgs
} from 'hooklib/lib'
import { exit } from 'process'
import {
cleanupJob,
prepareJob,
runContainerStep,
runScriptStep
} from './hooks'
async function run(): Promise<void> {
const input = await getInputFromStdin()
const args = input['args']
const command = input['command']
const responseFile = input['responseFile']
const state = input['state']
try {
switch (command) {
case Command.PrepareJob:
await prepareJob(args as PrepareJobArgs, responseFile)
return exit(0)
case Command.CleanupJob:
await cleanupJob(null, state, null)
return exit(0)
case Command.RunScriptStep:
await runScriptStep(args as RunScriptStepArgs, state)
return exit(0)
case Command.RunContainerStep:
await runContainerStep(args as RunContainerStepArgs, state)
return exit(0)
default:
throw new Error(`Command not recognized: ${command}`)
}
} catch (error) {
core.error(`${error}`)
exit(1)
}
}
void run()

View File

@@ -0,0 +1,56 @@
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
/* eslint-disable import/no-commonjs */
import * as core from '@actions/core'
// Import this way otherwise typescript has errors
const exec = require('@actions/exec')
export interface RunDockerCommandOptions {
workingDir?: string
input?: Buffer
}
export async function runDockerCommand(
args: string[],
options?: RunDockerCommandOptions
): Promise<string> {
const pipes = await exec.getExecOutput('docker', args, options)
if (pipes.exitCode !== 0) {
core.error(`Docker failed with exit code ${pipes.exitCode}`)
return Promise.reject(pipes.stderr)
}
return Promise.resolve(pipes.stdout)
}
export function sanitize(val: string): string {
if (!val || typeof val !== 'string') {
return ''
}
const newNameBuilder: string[] = []
for (let i = 0; i < val.length; i++) {
const char = val.charAt(i)
if (!newNameBuilder.length) {
if (isAlpha(char)) {
newNameBuilder.push(char)
}
} else {
if (isAlpha(char) || isNumeric(char) || char === '_') {
newNameBuilder.push(char)
}
}
}
return newNameBuilder.join('')
}
// isAlpha accepts single character and checks if
// that character is [a-zA-Z]
function isAlpha(val: string): boolean {
return (
val.length === 1 &&
((val >= 'a' && val <= 'z') || (val >= 'A' && val <= 'Z'))
)
}
function isNumeric(val: string): boolean {
return val.length === 1 && val >= '0' && val <= '9'
}