Reduce the amount of data copied to the workflow pod (#293)

* run script copies back only runner file commands

* wip

* fix

* fmt

* user volume mount

* try doing only file commands

* typo

* remove _temp_pre

* Update packages/k8s/src/hooks/run-script-step.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update packages/k8s/src/hooks/run-script-step.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* better escape

* no useless escapes

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

packages/k8s/src/hooks/run-script-step.ts

@@ -23,25 +23,51 @@ export async function runScriptStep(
   )
 
   const workdir = dirname(process.env.RUNNER_WORKSPACE as string)
-  const containerTemp = '/__w/_temp'
   const runnerTemp = `${workdir}/_temp`
-  await execCpToPod(state.jobPod, runnerTemp, containerTemp)
+  const containerTemp = '/__w/_temp'
+  const containerTempSrc = '/__w/_temp_pre'
+  // Ensure base and staging dirs exist before copying
+  await execPodStep(
+    [
+      'sh',
+      '-c',
+      'mkdir -p /__w && mkdir -p /__w/_temp && mkdir -p /__w/_temp_pre'
+    ],
+    state.jobPod,
+    JOB_CONTAINER_NAME
+  )
+  await execCpToPod(state.jobPod, runnerTemp, containerTempSrc)
 
   // Copy GitHub directories from temp to /github
-  const setupCommands = [
+  // Merge strategy:
-    'mkdir -p /github',
+  // - Overwrite files in _runner_file_commands
-    'cp -r /__w/_temp/_github_home /github/home',
+  // - Append files not already present elsewhere
-    'cp -r /__w/_temp/_github_workflow /github/workflow'
+  const mergeCommands = [
+    'set -e',
+    'mkdir -p /__w/_temp /__w/_temp_pre',
+    'SRC=/__w/_temp_pre',
+    'DST=/__w/_temp',
+    // Overwrite _runner_file_commands
+    `find "$SRC" -type f ! -path "*/_runner_file_commands/*" -exec sh -c '
+    rel="\${1#$2/}"
+    target="$3/$rel"
+    mkdir -p "$(dirname "$target")"
+    cp -a "$1" "$target"
+  ' _ {} "$SRC" "$DST" \\;`,
+    // Remove _temp_pre after merging
+    'rm -rf /__w/_temp_pre'
   ]
 
   try {
     await execPodStep(
-      ['sh', '-c', shlex.quote(setupCommands.join(' && '))],
+      ['sh', '-c', mergeCommands.join(' && ')],
       state.jobPod,
       JOB_CONTAINER_NAME
     )
   } catch (err) {
-    core.debug(`Failed to copy GitHub directories: ${JSON.stringify(err)}`)
+    core.debug(`Failed to merge temp directories: ${JSON.stringify(err)}`)
+    const message = (err as any)?.response?.body?.message || err
+    throw new Error(`failed to merge temp dirs: ${message}`)
   }
 
   // Execute the entrypoint script
@@ -69,7 +95,11 @@ export async function runScriptStep(
     core.debug(
       `Copying from job pod '${state.jobPod}' ${containerTemp} to ${runnerTemp}`
     )
-    await execCpFromPod(state.jobPod, containerTemp, workdir)
+    await execCpFromPod(
+      state.jobPod,
+      `${containerTemp}/_runner_file_commands`,
+      `${workdir}/_temp`
+    )
   } catch (error) {
     core.warning('Failed to copy _temp from pod')
   }

packages/k8s/src/k8s/index.ts

@@ -831,7 +831,7 @@ export async function isPodContainerAlpine(
       [
         'sh',
         '-c',
-        `'[ $(cat /etc/*release* | grep -i -e "^ID=*alpine*" -c) != 0 ] || exit 1'`
+        `[ $(cat /etc/*release* | grep -i -e "^ID=*alpine*" -c) != 0 ] || exit 1`
       ],
       podName,
       containerName

packages/k8s/src/k8s/utils.ts

@@ -288,6 +288,11 @@ function mergeLists<T>(base?: T[], from?: T[]): T[] {
 }
 
 export function fixArgs(args: string[]): string[] {
+  // Preserve shell command strings passed via `sh -c` without re-tokenizing.
+  // Retokenizing would split the script into multiple args, breaking `sh -c`.
+  if (args.length >= 2 && args[0] === 'sh' && args[1] === '-c') {
+    return args
+  }
   return shlex.split(args.join(' '))
 }
 

packages/k8s/tests/prepare-job-test.ts

@@ -45,7 +45,7 @@ describe('Prepare job', () => {
       process.env.GITHUB_WORKSPACE as string,
       'myvolume'
     )
-    fs.mkdirSync(userVolumeMount)
+    fs.mkdirSync(userVolumeMount, { recursive: true })
     fs.writeFileSync(path.join(userVolumeMount, 'file.txt'), 'hello')
     prepareJobData.args.container.userMountVolumes = [
       {
@@ -63,11 +63,7 @@ describe('Prepare job', () => {
     )
 
     await execPodStep(
-      [
+      ['sh', '-c', '[ "$(cat /__w/myvolume/file.txt)" = "hello" ] || exit 5'],
-        'sh',
-        '-c',
-        '\'[ "$(cat /__w/myvolume/file.txt)" = "hello" ] || exit 5\''
-      ],
       content!.state!.jobPod,
       JOB_CONTAINER_NAME
     ).then(output => {