podman

src/Misc/containerEngineHandlers/podmanHandler/.eslintignore

@@ -0,0 +1,3 @@
+dist/
+lib/
+node_modules/

src/Misc/containerEngineHandlers/podmanHandler/.eslintrc.json

@@ -0,0 +1,59 @@
+{
+    "plugins": ["jest", "@typescript-eslint"],
+    "extends": ["plugin:github/es6"],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+      "ecmaVersion": 9,
+      "sourceType": "module",
+      "project": "./tsconfig.json"
+    },
+    "rules": {
+      "eslint-comments/no-use": "off",
+      "import/no-namespace": "off",
+      "no-console": "off",
+      "no-unused-vars": "off",
+      "@typescript-eslint/no-unused-vars": "error",
+      "@typescript-eslint/explicit-member-accessibility": ["error", {"accessibility": "no-public"}],
+      "@typescript-eslint/no-require-imports": "error",
+      "@typescript-eslint/array-type": "error",
+      "@typescript-eslint/await-thenable": "error",
+      "@typescript-eslint/ban-ts-ignore": "error",
+      "camelcase": "off",
+      "@typescript-eslint/camelcase": "error",
+      "@typescript-eslint/class-name-casing": "error",
+      "@typescript-eslint/explicit-function-return-type": ["error", {"allowExpressions": true}],
+      "@typescript-eslint/func-call-spacing": ["error", "never"],
+      "@typescript-eslint/generic-type-naming": ["error", "^[A-Z][A-Za-z]*$"],
+      "@typescript-eslint/no-array-constructor": "error",
+      "@typescript-eslint/no-empty-interface": "error",
+      "@typescript-eslint/no-explicit-any": "error",
+      "@typescript-eslint/no-extraneous-class": "error",
+      "@typescript-eslint/no-for-in-array": "error",
+      "@typescript-eslint/no-inferrable-types": "error",
+      "@typescript-eslint/no-misused-new": "error",
+      "@typescript-eslint/no-namespace": "error",
+      "@typescript-eslint/no-non-null-assertion": "warn",
+      "@typescript-eslint/no-object-literal-type-assertion": "error",
+      "@typescript-eslint/no-unnecessary-qualifier": "error",
+      "@typescript-eslint/no-unnecessary-type-assertion": "error",
+      "@typescript-eslint/no-useless-constructor": "error",
+      "@typescript-eslint/no-var-requires": "error",
+      "@typescript-eslint/prefer-for-of": "warn",
+      "@typescript-eslint/prefer-function-type": "warn",
+      "@typescript-eslint/prefer-includes": "error",
+      "@typescript-eslint/prefer-interface": "error",
+      "@typescript-eslint/prefer-string-starts-ends-with": "error",
+      "@typescript-eslint/promise-function-async": "error",
+      "@typescript-eslint/require-array-sort-compare": "error",
+      "@typescript-eslint/restrict-plus-operands": "error",
+      "semi": "off",
+      "@typescript-eslint/semi": ["error", "never"],
+      "@typescript-eslint/type-annotation-spacing": "error",
+      "@typescript-eslint/unbound-method": "error"
+    },
+    "env": {
+      "node": true,
+      "es6": true,
+      "jest/globals": true
+    }
+  }

src/Misc/containerEngineHandlers/podmanHandler/.prettierignore

@@ -0,0 +1,3 @@
+dist/
+lib/
+node_modules/

src/Misc/containerEngineHandlers/podmanHandler/.prettierrc.json

@@ -0,0 +1,11 @@
+{
+    "printWidth": 80,
+    "tabWidth": 2,
+    "useTabs": false,
+    "semi": false,
+    "singleQuote": true,
+    "trailingComma": "none",
+    "bracketSpacing": false,
+    "arrowParens": "avoid",
+    "parser": "typescript"
+  }

src/Misc/containerEngineHandlers/podmanHandler/README.md

@@ -0,0 +1 @@
+To update podmanHandler under `Misc/layoutbin` run `npm install && npm run all`

src/Misc/containerEngineHandlers/podmanHandler/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "podmanHandler",
+  "version": "1.0.0",
+  "description": "GitHub Actions",
+  "main": "lib/podmanHandler.js",
+  "scripts": {
+    "build": "tsc",
+    "format": "prettier --write **/*.ts",
+    "format-check": "prettier --check **/*.ts",
+    "lint": "eslint src/**/*.ts",
+    "pack": "ncc build -o ../../layoutbin/podmanHandler",
+    "all": "npm run build && npm run format && npm run lint && npm run pack"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/actions/runner.git"
+  },
+  "keywords": [
+    "actions"
+  ],
+  "author": "GitHub Actions",
+  "license": "MIT",
+  "dependencies": {
+    "@actions/exec": "^1.1.0",
+    "@actions/core": "^1.6.0"
+  },
+  "devDependencies": {
+    "@types/node": "^12.7.12",
+    "@typescript-eslint/parser": "^2.8.0",
+    "@zeit/ncc": "^0.20.5",
+    "eslint": "^6.8.0",
+    "eslint-plugin-github": "^2.0.0",
+    "prettier": "^1.19.1",
+    "typescript": "^3.6.4"
+  }
+}

src/Misc/containerEngineHandlers/podmanHandler/src/podmanHandler.ts

@@ -0,0 +1,89 @@
+import * as exec from '@actions/exec'
+import * as core from '@actions/core'
+import * as events from 'events'
+import * as readline from 'readline'
+
+async function run(): Promise<void> {
+  let input = ''
+
+  const rl = readline.createInterface({
+    input: process.stdin
+  })
+
+  rl.on('line', line => {
+    core.debug(`Line from STDIN: ${line}`)
+    input = line
+  })
+
+  await events.once(rl, 'close')
+
+  core.debug(input)
+
+  const inputJson = JSON.parse(input)
+  core.debug(JSON.stringify(inputJson))
+
+  const command = inputJson.command
+  if (command === 'Create') {
+    const creationInput = inputJson.creationInput
+    core.debug(JSON.stringify(creationInput))
+    const containers = creationInput.containers
+    const jobContainer = containers[0]
+
+    const networkName = 'actions_podman_network'
+    // podman network create {network}  -> track and return `network` for ${{job.container.network}}
+    await exec.exec('podman', ['network', 'create', networkName])
+
+    const containerImage = `docker.io/library/${jobContainer.containerImage}`
+    // podman pull docker.io/library/{image}
+    await exec.exec('podman', ['pull', containerImage])
+
+    // podman create --name e088c842be1f46b394212618408aaba0_node1016jessie_6196c9
+    //      --label fa4e14
+    //      --workdir /__w/canary/canary
+    //      --network github_network_f98a6e1e96e74d919d814c165641cba3
+    //      -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true
+    //      -v "/var/run/docker.sock":"/var/run/docker.sock"
+    //      -v "/home/runner/work":"/__w"
+    //      -v "/home/runner/runners/2.283.2/externals":"/__e":ro
+    //      -v "/home/runner/work/_temp":"/__w/_temp"
+    //      -v "/home/runner/work/_actions":"/__w/_actions"
+    //      -v "/opt/hostedtoolcache":"/__t"
+    //      -v "/home/runner/work/_temp/_github_home":"/github/home"
+    //      -v "/home/runner/work/_temp/_github_workflow":"/github/workflow"
+    //      --entrypoint "tail" node:10.16-jessie "-f" "/dev/null"
+    const containerId = await exec.getExecOutput('podman', [
+      'create',
+      `--workdir ${jobContainer.containerWorkDirectory}`,
+      `--network ${networkName}`,
+      `-v /Users/ting/Desktop/runner/_layout/_work:/__w`,
+      `--entrypoint "${jobContainer.containerEntryPoint}"`,
+      `${containerImage}`,
+      `${jobContainer.containerEntryPointArgs}`
+    ])
+
+    core.debug(JSON.stringify(containerId))
+
+    // podman start {containerId}
+    await exec.exec('podman', ['start', containerId.stdout])
+
+    // get PATH inside the container
+
+    // output containerId for ${{job.container.id}}
+
+    const creationOutput = {
+      JobContainerId: containerId.stdout,
+      Network: networkName
+    }
+
+    const output = JSON.stringify({CreationOutput: creationOutput})
+    core.debug(output)
+
+    process.stderr.write(output)
+  }
+  // else if (command === 'Remove') {
+  // } else if (command === 'Exec') {
+  // }
+  await exec.exec('podman', ['network', 'ls'])
+}
+
+run()

src/Misc/containerEngineHandlers/podmanHandler/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "compilerOptions": {
+    "target": "es6",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    "outDir": "./lib",                        /* Redirect output structure to the directory. */
+    "rootDir": "./src",                       /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
+    "strict": true,                           /* Enable all strict type-checking options. */
+    "noImplicitAny": true,                    /* Raise error on expressions and declarations with an implied 'any' type. */
+    "esModuleInterop": true                   /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
+  },
+  "exclude": ["node_modules", "**/*.test.ts"]
+}

src/Misc/layoutbin/podman-handler.js

@@ -0,0 +1,49 @@
+// Job container creation
+
+// podman network create {network}  -> track and return `network` for ${{job.container.network}}
+
+// podman pull docker.io/library/{image}
+
+// podman create --name e088c842be1f46b394212618408aaba0_node1016jessie_6196c9 
+//      --label fa4e14 
+//      --workdir /__w/canary/canary 
+//      --network github_network_f98a6e1e96e74d919d814c165641cba3  
+//      -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true 
+//      -v "/var/run/docker.sock":"/var/run/docker.sock" 
+//      -v "/home/runner/work":"/__w" 
+//      -v "/home/runner/runners/2.283.2/externals":"/__e":ro 
+//      -v "/home/runner/work/_temp":"/__w/_temp" 
+//      -v "/home/runner/work/_actions":"/__w/_actions" 
+//      -v "/opt/hostedtoolcache":"/__t" 
+//      -v "/home/runner/work/_temp/_github_home":"/github/home" 
+//      -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" 
+//      --entrypoint "tail" node:10.16-jessie "-f" "/dev/null"
+
+// podman start {containerId}
+
+// get PATH inside the container
+
+// output containerId for ${{job.container.id}}
+
+
+
+// Job container stop
+
+// podman rm --force {containerId}
+
+// podman network rm {network}
+
+
+// Run step
+
+// podman exec -i --workdir /__w/canary/canary 
+// -e GITHUB_JOB -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY 
+// -e GITHUB_REPOSITORY_OWNER -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER 
+// -e GITHUB_RETENTION_DAYS -e GITHUB_RUN_ATTEMPT -e GITHUB_ACTOR 
+// -e GITHUB_WORKFLOW -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GITHUB_EVENT_NAME 
+// -e GITHUB_SERVER_URL -e GITHUB_API_URL -e GITHUB_GRAPHQL_URL 
+// -e GITHUB_WORKSPACE -e GITHUB_ACTION -e GITHUB_EVENT_PATH -e GITHUB_ACTION_REPOSITORY 
+// -e GITHUB_ACTION_REF -e GITHUB_PATH -e GITHUB_ENV -e RUNNER_DEBUG 
+// -e RUNNER_OS -e RUNNER_NAME -e RUNNER_TOOL_CACHE 
+// -e RUNNER_TEMP -e RUNNER_WORKSPACE 
+//   eccdf520697a035599d6e8c8dc801f004fdd3797cdce88f590aba3669a88d9bc sh -e /__w/_temp/d3b30383-719c-4e76-a16f-8f85443352be.sh

src/Runner.Worker/ContainerOperationProvider.cs

@@ -12,9 +12,61 @@ using GitHub.Runner.Sdk;
 using GitHub.DistributedTask.Pipelines.ContextData;
 using Microsoft.Win32;
 using GitHub.DistributedTask.Pipelines.ObjectTemplating;
+using System.Threading.Channels;
+using GitHub.Services.WebApi;
+using System.Text;
+using System.Runtime.Serialization;
 
 namespace GitHub.Runner.Worker
 {
+    [DataContract]
+    public class ContainerEngineHandlerInput
+    {
+        [DataMember]
+        public string Command { get; set; }
+
+        [DataMember]
+        public ContainersCreationInput CreationInput { get; set; }
+
+        [DataMember]
+        public object JobContainerExecInput { get; set; }
+
+        [DataMember]
+        public ContainersRemoveInput RemoveInput { get; set; }
+    }
+
+    [DataContract]
+    public class ContainersCreationInput
+    {
+        [DataMember]
+        public List<ContainerInfo> Containers { get; set; }
+    }
+
+    [DataContract]
+    public class ContainersRemoveInput
+    {
+        [DataMember]
+        public string Network { get; set; }
+        [DataMember]
+        public string JobContainerId { get; set; }
+    }
+
+    [DataContract]
+    public class ContainersCreationOutput
+    {
+        [DataMember]
+        public string Network { get; set; }
+        [DataMember]
+        public string JobContainerId { get; set; }
+    }
+
+    [DataContract]
+    public class ContainerEngineHandlerOutput
+    {
+        [DataMember]
+        public ContainersCreationOutput CreationOutput { get; set; }
+    }
+
     [ServiceLocator(Default = typeof(ContainerOperationProvider))]
     public interface IContainerOperationProvider : IRunnerService
     {
@@ -35,14 +87,46 @@ namespace GitHub.Runner.Worker
         public async Task StartContainersAsync(IExecutionContext executionContext, object data)
         {
             Trace.Entering();
-            if (!Constants.Runner.Platform.Equals(Constants.OSPlatform.Linux))
-            {
-                throw new NotSupportedException("Container operations are only supported on Linux runners");
-            }
+            // if (!Constants.Runner.Platform.Equals(Constants.OSPlatform.Linux))
+            // {
+            //     throw new NotSupportedException("Container operations are only supported on Linux runners");
+            // }
             ArgUtil.NotNull(executionContext, nameof(executionContext));
             List<ContainerInfo> containers = data as List<ContainerInfo>;
             ArgUtil.NotNull(containers, nameof(containers));
 
+            foreach (var container in containers)
+            {
+                if (container.IsJobContainer)
+                {
+                    // Configure job container - Mount workspace and tools, set up environment, and start long running process
+                    var githubContext = executionContext.ExpressionValues["github"] as GitHubContext;
+                    ArgUtil.NotNull(githubContext, nameof(githubContext));
+                    var workingDirectory = githubContext["workspace"] as StringContextData;
+                    ArgUtil.NotNullOrEmpty(workingDirectory, nameof(workingDirectory));
+                    container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work))));
+                    container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true));
+                    container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp))));
+                    container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Actions), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Actions))));
+                    container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
+
+                    var tempHomeDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Temp), "_github_home");
+                    Directory.CreateDirectory(tempHomeDirectory);
+                    container.MountVolumes.Add(new MountVolume(tempHomeDirectory, "/github/home"));
+                    container.AddPathTranslateMapping(tempHomeDirectory, "/github/home");
+                    container.ContainerEnvironmentVariables["HOME"] = container.TranslateToContainerPath(tempHomeDirectory);
+
+                    var tempWorkflowDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Temp), "_github_workflow");
+                    Directory.CreateDirectory(tempWorkflowDirectory);
+                    container.MountVolumes.Add(new MountVolume(tempWorkflowDirectory, "/github/workflow"));
+                    container.AddPathTranslateMapping(tempWorkflowDirectory, "/github/workflow");
+
+                    container.ContainerWorkDirectory = container.TranslateToContainerPath(workingDirectory);
+                    container.ContainerEntryPoint = "tail";
+                    container.ContainerEntryPointArgs = "\"-f\" \"/dev/null\"";
+                }
+            }
+
             var postJobStep = new JobExtensionRunner(runAsync: this.StopContainersAsync,
                                                 condition: $"{PipelineTemplateConstants.Always}()",
                                                 displayName: "Stop containers",
@@ -51,9 +135,70 @@ namespace GitHub.Runner.Worker
             executionContext.Debug($"Register post job cleanup for stopping/deleting containers.");
             executionContext.RegisterPostJobStep(postJobStep);
 
-            // Check whether we are inside a container.
-            // Our container feature requires to map working directory from host to the container.
-            // If we are already inside a container, we will not able to find out the real working direcotry path on the host.
+            var podManHandler = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "podmanHandler", "index.js");
+            if (File.Exists(podManHandler))
+            {
+                var podmanInput = new ContainerEngineHandlerInput()
+                {
+                    Command = "Create",
+                    CreationInput = new ContainersCreationInput()
+                    {
+                        Containers = containers
+                    }
+                };
+
+                ContainerEngineHandlerOutput podmanOutput = null;
+                using (var processInvoker = HostContext.CreateService<IProcessInvoker>())
+                {
+                    var redirectStandardIn = Channel.CreateUnbounded<string>(new UnboundedChannelOptions() { SingleReader = true, SingleWriter = true });
+                    redirectStandardIn.Writer.TryWrite(JsonUtility.ToString(podmanInput));
+
+                    processInvoker.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs message)
+                    {
+                        executionContext.Output(message.Data);
+                    };
+
+                    processInvoker.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs message)
+                    {
+                        executionContext.Output(message.Data);
+                        if (podmanOutput == null)
+                        {
+                            try
+                            {
+                                podmanOutput = JsonUtility.FromString<ContainerEngineHandlerOutput>(message.Data);
+                            }
+                            catch (Exception ex)
+                            {
+                                executionContext.Error(ex);
+                            }
+                        }
+                    };
+
+                    // Execute the process. Exit code 0 should always be returned.
+                    // A non-zero exit code indicates infrastructural failure.
+                    // Task failure should be communicated over STDOUT using ## commands.
+                    await processInvoker.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Work),
+                                                      fileName: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node12", "bin", $"node{IOUtil.ExeExtension}"),
+                                                      arguments: podManHandler,
+                                                      environment: null,
+                                                      requireExitCodeZero: false,
+                                                      outputEncoding: Encoding.UTF8,
+                                                      killProcessOnCancel: false,
+                                                      redirectStandardIn: redirectStandardIn,
+                                                      cancellationToken: executionContext.CancellationToken);
+                }
+
+                if (podmanOutput != null)
+                {
+                    executionContext.JobContext.Container["network"] = new StringContextData(podmanOutput.CreationOutput.Network);
+                    executionContext.JobContext.Container["id"] = new StringContextData(podmanOutput.CreationOutput.JobContainerId);
+                }
+            }
+            else
+            {
+                // Check whether we are inside a container.
+                // Our container feature requires to map working directory from host to the container.
+                // If we are already inside a container, we will not able to find out the real working direcotry path on the host.
 #if OS_WINDOWS
             // service CExecSvc is Container Execution Agent.
             ServiceController[] scServices = ServiceController.GetServices();
@@ -62,11 +207,11 @@ namespace GitHub.Runner.Worker
                 throw new NotSupportedException("Container feature is not supported when runner is already running inside container.");
             }
 #else
-            var initProcessCgroup = File.ReadLines("/proc/1/cgroup");
-            if (initProcessCgroup.Any(x => x.IndexOf(":/docker/", StringComparison.OrdinalIgnoreCase) >= 0))
-            {
-                throw new NotSupportedException("Container feature is not supported when runner is already running inside container.");
-            }
+                var initProcessCgroup = File.ReadLines("/proc/1/cgroup");
+                if (initProcessCgroup.Any(x => x.IndexOf(":/docker/", StringComparison.OrdinalIgnoreCase) >= 0))
+                {
+                    throw new NotSupportedException("Container feature is not supported when runner is already running inside container.");
+                }
 #endif
 
 #if OS_WINDOWS
@@ -90,68 +235,69 @@ namespace GitHub.Runner.Worker
             }
 #endif
 
-            // Check docker client/server version
-            executionContext.Output("##[group]Checking docker version");
-            DockerVersion dockerVersion = await _dockerManager.DockerVersion(executionContext);
-            executionContext.Output("##[endgroup]");
+                // Check docker client/server version
+                executionContext.Output("##[group]Checking docker version");
+                DockerVersion dockerVersion = await _dockerManager.DockerVersion(executionContext);
+                executionContext.Output("##[endgroup]");
 
-            ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion));
-            ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion));
+                ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion));
+                ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion));
 
 #if OS_WINDOWS
             Version requiredDockerEngineAPIVersion = new Version(1, 30);  // Docker-EE version 17.6
 #else
-            Version requiredDockerEngineAPIVersion = new Version(1, 35); // Docker-CE version 17.12
+                Version requiredDockerEngineAPIVersion = new Version(1, 35); // Docker-CE version 17.12
 #endif
 
-            if (dockerVersion.ServerVersion < requiredDockerEngineAPIVersion)
-            {
-                throw new NotSupportedException($"Min required docker engine API server version is '{requiredDockerEngineAPIVersion}', your docker ('{_dockerManager.DockerPath}') server version is '{dockerVersion.ServerVersion}'");
-            }
-            if (dockerVersion.ClientVersion < requiredDockerEngineAPIVersion)
-            {
-                throw new NotSupportedException($"Min required docker engine API client version is '{requiredDockerEngineAPIVersion}', your docker ('{_dockerManager.DockerPath}') client version is '{dockerVersion.ClientVersion}'");
-            }
-
-            // Clean up containers left by previous runs
-            executionContext.Output("##[group]Clean up resources from previous jobs");
-            var staleContainers = await _dockerManager.DockerPS(executionContext, $"--all --quiet --no-trunc --filter \"label={_dockerManager.DockerInstanceLabel}\"");
-            foreach (var staleContainer in staleContainers)
-            {
-                int containerRemoveExitCode = await _dockerManager.DockerRemove(executionContext, staleContainer);
-                if (containerRemoveExitCode != 0)
+                if (dockerVersion.ServerVersion < requiredDockerEngineAPIVersion)
                 {
-                    executionContext.Warning($"Delete stale containers failed, docker rm fail with exit code {containerRemoveExitCode} for container {staleContainer}");
+                    throw new NotSupportedException($"Min required docker engine API server version is '{requiredDockerEngineAPIVersion}', your docker ('{_dockerManager.DockerPath}') server version is '{dockerVersion.ServerVersion}'");
+                }
+                if (dockerVersion.ClientVersion < requiredDockerEngineAPIVersion)
+                {
+                    throw new NotSupportedException($"Min required docker engine API client version is '{requiredDockerEngineAPIVersion}', your docker ('{_dockerManager.DockerPath}') client version is '{dockerVersion.ClientVersion}'");
                 }
-            }
 
-            int networkPruneExitCode = await _dockerManager.DockerNetworkPrune(executionContext);
-            if (networkPruneExitCode != 0)
-            {
-                executionContext.Warning($"Delete stale container networks failed, docker network prune fail with exit code {networkPruneExitCode}");
-            }
-            executionContext.Output("##[endgroup]");
+                // Clean up containers left by previous runs
+                executionContext.Output("##[group]Clean up resources from previous jobs");
+                var staleContainers = await _dockerManager.DockerPS(executionContext, $"--all --quiet --no-trunc --filter \"label={_dockerManager.DockerInstanceLabel}\"");
+                foreach (var staleContainer in staleContainers)
+                {
+                    int containerRemoveExitCode = await _dockerManager.DockerRemove(executionContext, staleContainer);
+                    if (containerRemoveExitCode != 0)
+                    {
+                        executionContext.Warning($"Delete stale containers failed, docker rm fail with exit code {containerRemoveExitCode} for container {staleContainer}");
+                    }
+                }
 
-            // Create local docker network for this job to avoid port conflict when multiple runners run on same machine.
-            // All containers within a job join the same network
-            executionContext.Output("##[group]Create local container network");
-            var containerNetwork = $"github_network_{Guid.NewGuid().ToString("N")}";
-            await CreateContainerNetworkAsync(executionContext, containerNetwork);
-            executionContext.JobContext.Container["network"] = new StringContextData(containerNetwork);
-            executionContext.Output("##[endgroup]");
+                int networkPruneExitCode = await _dockerManager.DockerNetworkPrune(executionContext);
+                if (networkPruneExitCode != 0)
+                {
+                    executionContext.Warning($"Delete stale container networks failed, docker network prune fail with exit code {networkPruneExitCode}");
+                }
+                executionContext.Output("##[endgroup]");
 
-            foreach (var container in containers)
-            {
-                container.ContainerNetwork = containerNetwork;
-                await StartContainerAsync(executionContext, container);
-            }
+                // Create local docker network for this job to avoid port conflict when multiple runners run on same machine.
+                // All containers within a job join the same network
+                executionContext.Output("##[group]Create local container network");
+                var containerNetwork = $"github_network_{Guid.NewGuid().ToString("N")}";
+                await CreateContainerNetworkAsync(executionContext, containerNetwork);
+                executionContext.JobContext.Container["network"] = new StringContextData(containerNetwork);
+                executionContext.Output("##[endgroup]");
 
-            executionContext.Output("##[group]Waiting for all services to be ready");
-            foreach (var container in containers.Where(c => !c.IsJobContainer))
-            {
-                await ContainerHealthcheck(executionContext, container);
+                foreach (var container in containers)
+                {
+                    container.ContainerNetwork = containerNetwork;
+                    await StartContainerAsync(executionContext, container);
+                }
+
+                executionContext.Output("##[group]Waiting for all services to be ready");
+                foreach (var container in containers.Where(c => !c.IsJobContainer))
+                {
+                    await ContainerHealthcheck(executionContext, container);
+                }
+                executionContext.Output("##[endgroup]");
             }
-            executionContext.Output("##[endgroup]");
         }
 
         public async Task StopContainersAsync(IExecutionContext executionContext, object data)
@@ -162,12 +308,69 @@ namespace GitHub.Runner.Worker
             List<ContainerInfo> containers = data as List<ContainerInfo>;
             ArgUtil.NotNull(containers, nameof(containers));
 
-            foreach (var container in containers)
+            var podManHandler = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "podmanHandler", "index.js");
+            if (File.Exists(podManHandler))
             {
-                await StopContainerAsync(executionContext, container);
+                var podmanInput = new ContainerEngineHandlerInput()
+                {
+                    Command = "Remove",
+                    RemoveInput = new ContainersRemoveInput()
+                    {
+                        Network = executionContext.JobContext.Container["network"].ToString(),
+                        JobContainerId = executionContext.JobContext.Container["id"].ToString()
+                    }
+                };
+
+                ContainerEngineHandlerOutput podmanOutput = null;
+                using (var processInvoker = HostContext.CreateService<IProcessInvoker>())
+                {
+                    var redirectStandardIn = Channel.CreateUnbounded<string>(new UnboundedChannelOptions() { SingleReader = true, SingleWriter = true });
+                    redirectStandardIn.Writer.TryWrite(JsonUtility.ToString(podmanInput));
+
+                    processInvoker.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs message)
+                    {
+                        executionContext.Output(message.Data);
+                    };
+
+                    processInvoker.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs message)
+                    {
+                        executionContext.Output(message.Data);
+                        if (podmanOutput == null)
+                        {
+                            try
+                            {
+                                podmanOutput = JsonUtility.FromString<ContainerEngineHandlerOutput>(message.Data);
+                            }
+                            catch (Exception ex)
+                            {
+                                executionContext.Error(ex);
+                            }
+                        }
+                    };
+
+                    // Execute the process. Exit code 0 should always be returned.
+                    // A non-zero exit code indicates infrastructural failure.
+                    // Task failure should be communicated over STDOUT using ## commands.
+                    await processInvoker.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Work),
+                                                      fileName: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node12", "bin", $"node{IOUtil.ExeExtension}"),
+                                                      arguments: podManHandler,
+                                                      environment: null,
+                                                      requireExitCodeZero: false,
+                                                      outputEncoding: Encoding.UTF8,
+                                                      killProcessOnCancel: false,
+                                                      redirectStandardIn: redirectStandardIn,
+                                                      cancellationToken: executionContext.CancellationToken);
+                }
+            }
+            else
+            {
+                foreach (var container in containers)
+                {
+                    await StopContainerAsync(executionContext, container);
+                }
+                // Remove the container network
+                await RemoveContainerNetworkAsync(executionContext, containers.First().ContainerNetwork);
             }
-            // Remove the container network
-            await RemoveContainerNetworkAsync(executionContext, containers.First().ContainerNetwork);
         }
 
         private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container)