temp tracing for client auth

- github/c2c-actions-support#883

.github/workflows/release.yml

@@ -260,6 +260,17 @@ jobs:
           console.log(releaseNote)
           core.setOutput('version', runnerVersion);
           core.setOutput('note', releaseNote);
+
+    - name: Validate Packages HASH
+      working-directory: _package
+      run: |
+        ls -l
+        echo "${{needs.build.outputs.win-x64-sha}}  actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip" | shasum -a 256 -c
+        echo "${{needs.build.outputs.osx-x64-sha}}  actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}.tar.gz" | shasum -a 256 -c
+        echo "${{needs.build.outputs.linux-x64-sha}}  actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}.tar.gz" | shasum -a 256 -c
+        echo "${{needs.build.outputs.linux-arm-sha}}  actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}.tar.gz" | shasum -a 256 -c
+        echo "${{needs.build.outputs.linux-arm64-sha}}  actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz" | shasum -a 256 -c
+
     # Create GitHub release
     - uses: actions/create-release@master
       id: createRelease

releaseNote.md

@@ -1,32 +1,8 @@
 ## Features
 
-- Make run.sh|cmd handle update without quitting so containers using them as entrypoints don't exit on update (#1646, #1633, #1708)
-- Add support for Step Summary (#1642, #1667, #1712) 
-- Pass jobId to the actionsDownloadInfo controller (#1639)
-- updated systemd svc.sh to accept custom service file (#1612) 
-- Add ability to specify runner group when creating service (#1675)
-- Prefer node16 over node12 when running internal scripts (#1621) 
-- Sending telemetry about actions usage. (#1688) 
-- Bump node12 version to latest (#1651)
-- Add internal to node version function and use better env var name (#1715)
-- Force JS Actions Node version to 16 if FF is on unless user opted out (#1716)
-
 ## Bugs
-- Fix windows console runner update crash (#1670) 
+- Fixed an issue where websockets failed to successfully close when posting log lines (#1790)
-- Retry policy for methods GetTenantCredential and GetJITRunnerTokenAsync (#1691)
-- Skip DeleteAgentSession when the acess token has been revoked. (#1692)
-- Repaired hashFiles call so if error was thrown, it was returned to process invoker (#1678)
-- Runner throws null ref exception when new line after EOF is missing (#1687)
-- Lets allow up to 150 characters for services on linux/mac (#1710) 
 
-## Misc
-
-- Added examples and aligned language within docs/checks/actions.md (#1664)
-- Problem with debugging on macOS M1 (#1625) 
-- Fix typo in hashFiles.ts. (#1672) 
-- Allow mocked updates for E2E testing (#1654)
-- Move JobTelemetry and StepsTelemetry into GlobalContext. (#1680) 
-- Fix inconsistency of outputs (both canceled and cancelled are used (#1624)
 
 ## Windows x64
 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.

src/Directory.Build.props

@@ -47,6 +47,11 @@
     <DefineConstants>$(DefineConstants);DEBUG</DefineConstants>
   </PropertyGroup>
 
+  <!-- Set USE_BROKER vars -->
+  <PropertyGroup Condition="'$(USE_BROKER)' == 'true'">
+    <DefineConstants>$(DefineConstants);USE_BROKER</DefineConstants>
+  </PropertyGroup>
+
   <!-- Set Treat tarnings as errors -->
   <PropertyGroup>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>

src/Misc/expressionFunc/hashFiles/.eslintrc.json

@@ -1,6 +1,6 @@
 {
-    "plugins": ["jest", "@typescript-eslint"],
+    "plugins": ["@typescript-eslint"],
-    "extends": ["plugin:github/es6"],
+    "extends": ["plugin:github/recommended"],
     "parser": "@typescript-eslint/parser",
     "parserOptions": {
       "ecmaVersion": 9,
@@ -17,13 +17,16 @@
       "@typescript-eslint/no-require-imports": "error",
       "@typescript-eslint/array-type": "error",
       "@typescript-eslint/await-thenable": "error",
-      "@typescript-eslint/ban-ts-ignore": "error",
+      "@typescript-eslint/naming-convention": [
+        "error",
+        {
+          "selector": "default",
+          "format": ["camelCase"]
+        }
+      ],
       "camelcase": "off",
-      "@typescript-eslint/camelcase": "error",
-      "@typescript-eslint/class-name-casing": "error",
       "@typescript-eslint/explicit-function-return-type": ["error", {"allowExpressions": true}],
       "@typescript-eslint/func-call-spacing": ["error", "never"],
-      "@typescript-eslint/generic-type-naming": ["error", "^[A-Z][A-Za-z]*$"],
       "@typescript-eslint/no-array-constructor": "error",
       "@typescript-eslint/no-empty-interface": "error",
       "@typescript-eslint/no-explicit-any": "error",
@@ -33,7 +36,6 @@
       "@typescript-eslint/no-misused-new": "error",
       "@typescript-eslint/no-namespace": "error",
       "@typescript-eslint/no-non-null-assertion": "warn",
-      "@typescript-eslint/no-object-literal-type-assertion": "error",
       "@typescript-eslint/no-unnecessary-qualifier": "error",
       "@typescript-eslint/no-unnecessary-type-assertion": "error",
       "@typescript-eslint/no-useless-constructor": "error",
@@ -41,19 +43,19 @@
       "@typescript-eslint/prefer-for-of": "warn",
       "@typescript-eslint/prefer-function-type": "warn",
       "@typescript-eslint/prefer-includes": "error",
-      "@typescript-eslint/prefer-interface": "error",
       "@typescript-eslint/prefer-string-starts-ends-with": "error",
       "@typescript-eslint/promise-function-async": "error",
       "@typescript-eslint/require-array-sort-compare": "error",
       "@typescript-eslint/restrict-plus-operands": "error",
-      "semi": "off",
       "@typescript-eslint/semi": ["error", "never"],
       "@typescript-eslint/type-annotation-spacing": "error",
-      "@typescript-eslint/unbound-method": "error"
+      "@typescript-eslint/unbound-method": "error",
+      "filenames/match-regex" : "off",
+      "github/no-then" : 1, // warning
+      "semi": "off"
     },
     "env": {
       "node": true,
-      "es6": true,
+      "es6": true
-      "jest/globals": true
     }
   }

src/Misc/expressionFunc/hashFiles/package.json

@@ -25,10 +25,10 @@
   },
   "devDependencies": {
     "@types/node": "^12.7.12",
-    "@typescript-eslint/parser": "^2.8.0",
+    "@typescript-eslint/parser": "^5.15.0",
     "@zeit/ncc": "^0.20.5",
-    "eslint": "^6.8.0",
+    "eslint": "^8.11.0",
-    "eslint-plugin-github": "^2.0.0",
+    "eslint-plugin-github": "^4.3.5",
     "prettier": "^1.19.1",
     "typescript": "^3.6.4"
   }

src/Misc/expressionFunc/hashFiles/src/hashFiles.ts

@@ -1,9 +1,9 @@
-import * as glob from '@actions/glob'
 import * as crypto from 'crypto'
 import * as fs from 'fs'
+import * as glob from '@actions/glob'
+import * as path from 'path'
 import * as stream from 'stream'
 import * as util from 'util'
-import * as path from 'path'
 
 async function run(): Promise<void> {
   // arg0 -> node

src/Misc/layoutbin/RunnerService.js

@@ -3,94 +3,135 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 var childProcess = require("child_process");
-var path = require("path")
+var path = require("path");
 
-var supported = ['linux', 'darwin']
+var supported = ["linux", "darwin"];
 
 if (supported.indexOf(process.platform) == -1) {
-    console.log('Unsupported platform: ' + process.platform);
+  console.log("Unsupported platform: " + process.platform);
-    console.log('Supported platforms are: ' + supported.toString());
+  console.log("Supported platforms are: " + supported.toString());
   process.exit(1);
 }
 
 var stopping = false;
 var listener = null;
 
+var exitServiceAfterNFailures = Number(
+  process.env.GITHUB_ACTIONS_SERVICE_EXIT_AFTER_N_FAILURES
+);
+
+if (exitServiceAfterNFailures <= 0) {
+  exitServiceAfterNFailures = NaN;
+}
+
+var consecutiveFailureCount = 0;
+
+var gracefulShutdown = function () {
+  console.log("Shutting down runner listener");
+  stopping = true;
+  if (listener) {
+    console.log("Sending SIGINT to runner listener to stop");
+    listener.kill("SIGINT");
+
+    console.log("Sending SIGKILL to runner listener");
+    setTimeout(() => listener.kill("SIGKILL"), 30000).unref();
+  }
+};
+
 var runService = function () {
-    var listenerExePath = path.join(__dirname, '../bin/Runner.Listener');
+  var listenerExePath = path.join(__dirname, "../bin/Runner.Listener");
   var interactive = process.argv[2] === "interactive";
 
   if (!stopping) {
     try {
       if (interactive) {
-                console.log('Starting Runner listener interactively');
+        console.log("Starting Runner listener interactively");
-                listener = childProcess.spawn(listenerExePath, ['run'], { env: process.env });
+        listener = childProcess.spawn(listenerExePath, ["run"], {
+          env: process.env,
+        });
       } else {
-                console.log('Starting Runner listener with startup type: service');
+        console.log("Starting Runner listener with startup type: service");
-                listener = childProcess.spawn(listenerExePath, ['run', '--startuptype', 'service'], { env: process.env });
+        listener = childProcess.spawn(
+          listenerExePath,
+          ["run", "--startuptype", "service"],
+          { env: process.env }
+        );
       }
 
       console.log(`Started listener process, pid: ${listener.pid}`);
 
-            listener.stdout.on('data', (data) => {
+      listener.stdout.on("data", (data) => {
-                process.stdout.write(data.toString('utf8'));
+        if (data.toString("utf8").includes("Listening for Jobs")) {
+          consecutiveFailureCount = 0;
+        }
+        process.stdout.write(data.toString("utf8"));
       });
 
-            listener.stderr.on('data', (data) => {
+      listener.stderr.on("data", (data) => {
-                process.stdout.write(data.toString('utf8'));
+        process.stdout.write(data.toString("utf8"));
       });
 
       listener.on("error", (err) => {
         console.log(`Runner listener fail to start with error ${err.message}`);
       });
 
-            listener.on('close', (code) => {
+      listener.on("close", (code) => {
         console.log(`Runner listener exited with error code ${code}`);
 
         if (code === 0) {
-                    console.log('Runner listener exit with 0 return code, stop the service, no retry needed.');
+          console.log(
+            "Runner listener exit with 0 return code, stop the service, no retry needed."
+          );
           stopping = true;
         } else if (code === 1) {
-                    console.log('Runner listener exit with terminated error, stop the service, no retry needed.');
+          console.log(
+            "Runner listener exit with terminated error, stop the service, no retry needed."
+          );
           stopping = true;
         } else if (code === 2) {
-                    console.log('Runner listener exit with retryable error, re-launch runner in 5 seconds.');
+          console.log(
-                } else if (code === 3) {
+            "Runner listener exit with retryable error, re-launch runner in 5 seconds."
-                    console.log('Runner listener exit because of updating, re-launch runner in 5 seconds.');
+          );
+          consecutiveFailureCount = 0;
+        } else if (code === 3 || code === 4) {
+          console.log(
+            "Runner listener exit because of updating, re-launch runner in 5 seconds."
+          );
+          consecutiveFailureCount = 0;
         } else {
-                    console.log('Runner listener exit with undefined return code, re-launch runner in 5 seconds.');
+          var messagePrefix = "Runner listener exit with undefined return code";
+          consecutiveFailureCount++;
+          if (
+            !isNaN(exitServiceAfterNFailures) &&
+            consecutiveFailureCount >= exitServiceAfterNFailures
+          ) {
+            console.error(
+              `${messagePrefix}, exiting service after ${consecutiveFailureCount} consecutive failures`
+            );
+            gracefulShutdown();
+            return;
+          } else {
+            console.log(`${messagePrefix}, re-launch runner in 5 seconds.`);
+          }
         }
 
         if (!stopping) {
           setTimeout(runService, 5000);
         }
       });
-
     } catch (ex) {
       console.log(ex);
     }
   }
-}
+};
 
 runService();
-console.log('Started running service');
+console.log("Started running service");
 
-var gracefulShutdown = function (code) {
+process.on("SIGINT", () => {
-    console.log('Shutting down runner listener');
+  gracefulShutdown();
-    stopping = true;
-    if (listener) {
-        console.log('Sending SIGINT to runner listener to stop');
-        listener.kill('SIGINT');
-
-        console.log('Sending SIGKILL to runner listener');
-        setTimeout(() => listener.kill('SIGKILL'), 30000).unref();
-    }
-}
-
-process.on('SIGINT', () => {
-    gracefulShutdown(0);
 });
 
-process.on('SIGTERM', () => {
+process.on("SIGTERM", () => {
-    gracefulShutdown(0);
+  gracefulShutdown();
 });

src/Misc/layoutbin/darwin.svc.sh.template

@@ -63,7 +63,7 @@ function install()
     mkdir -p "${log_path}" || failed "failed to create ${log_path}"
 
     echo Creating ${PLIST_PATH}
-    sed "s/{{User}}/${SUDO_USER:-$USER}/g; s/{{SvcName}}/$SVC_NAME/g; s@{{RunnerRoot}}@${RUNNER_ROOT}@g; s@{{UserHome}}@$HOME@g;" "${TEMPLATE_PATH}" > "${TEMP_PATH}" || failed "failed to create replacement temp file"
+    sed "s/{{User}}/${USER:-$SUDO_USER}/g; s/{{SvcName}}/$SVC_NAME/g; s@{{RunnerRoot}}@${RUNNER_ROOT}@g; s@{{UserHome}}@$HOME@g;" "${TEMPLATE_PATH}" > "${TEMP_PATH}" || failed "failed to create replacement temp file"
     mv "${TEMP_PATH}" "${PLIST_PATH}" || failed "failed to copy plist"
 
     # Since we started with sudo, runsvc.sh will be owned by root. Change this to current login user.

src/Misc/layoutbin/hashFiles/index.js

@@ -1557,12 +1557,12 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const glob = __importStar(__webpack_require__(281));
 const crypto = __importStar(__webpack_require__(417));
 const fs = __importStar(__webpack_require__(747));
+const glob = __importStar(__webpack_require__(281));
+const path = __importStar(__webpack_require__(622));
 const stream = __importStar(__webpack_require__(413));
 const util = __importStar(__webpack_require__(669));
-const path = __importStar(__webpack_require__(622));
 function run() {
     var e_1, _a;
     return __awaiter(this, void 0, void 0, function* () {

src/Misc/layoutbin/update.sh.template

@@ -30,13 +30,13 @@ date "+[%F %T-%4N] Waiting for $runnerprocessname ($runnerpid) to complete" >> "
 while [ -e /proc/$runnerpid ]
 do
     date "+[%F %T-%4N] Process $runnerpid still running" >> "$logfile" 2>&1
-    sleep 2
+    "$rootfolder"/safe_sleep.sh 2
 done
 date "+[%F %T-%4N] Process $runnerpid finished running" >> "$logfile" 2>&1
 
 # start re-organize folders
 date "+[%F %T-%4N] Sleep 1 more second to make sure process exited" >> "$logfile" 2>&1
-sleep 1
+"$rootfolder"/safe_sleep.sh 1
 
 # the folder structure under runner root will be
 # ./bin -> bin.2.100.0 (junction folder)

src/Misc/layoutroot/run-helper.sh.template

@@ -10,22 +10,6 @@ fi
 # Run
 shopt -s nocasematch
 
-safe_sleep() {
-    if [ ! -x "$(command -v sleep)" ]; then
-        if [ ! -x "$(command -v ping)" ]; then
-            COUNT="0"
-            while [[ $COUNT != 5000 ]]; do
-                echo "SLEEP" > /dev/null
-                COUNT=$[$COUNT+1]
-            done
-        else
-            ping -c 5 127.0.0.1 > /dev/null
-        fi
-    else
-        sleep 5
-    fi
-}
-
 SOURCE="${BASH_SOURCE[0]}"
 while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
     DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
@@ -44,17 +28,17 @@ elif [[ $returnCode == 1 ]]; then
     exit 0
 elif [[ $returnCode == 2 ]]; then
     echo "Runner listener exit with retryable error, re-launch runner in 5 seconds."
-    safe_sleep
+    "$DIR"/safe_sleep.sh 5
     exit 2
 elif [[ $returnCode == 3 ]]; then
     # Sleep 5 seconds to wait for the runner update process finish
     echo "Runner listener exit because of updating, re-launch runner in 5 seconds"
-    safe_sleep
+    "$DIR"/safe_sleep.sh 5
     exit 2
 elif [[ $returnCode == 4 ]]; then
     # Sleep 5 seconds to wait for the ephemeral runner update process finish
     echo "Runner listener exit because of updating, re-launch ephemeral runner in 5 seconds"
-    safe_sleep
+    "$DIR"/safe_sleep.sh 5
     exit 2
 else
     echo "Exiting with unknown error code: ${returnCode}"

src/Misc/layoutroot/safe_sleep.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+SECONDS=0
+while [[ $SECONDS != $1 ]]; do
+    :
+done

src/Runner.Common/BrokerServer.cs

@@ -0,0 +1,59 @@
+using System;
+using System.Collections.Generic;
+using GitHub.DistributedTask.WebApi;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.Runner.Common.Util;
+using GitHub.Services.WebApi;
+using GitHub.Services.Common;
+using GitHub.Runner.Sdk;
+using System.Net;
+using System.Net.Http;
+
+namespace GitHub.Runner.Common
+{
+    [ServiceLocator(Default = typeof(BrokerServer))]
+    public interface IBrokerServer : IRunnerService
+    {
+        Task ConnectAsync(Uri serverUrl, CancellationToken cancellationToken);
+        Task<string> GetMessageAsync(TaskAgentSession session, RunnerSettings settings, long? lastMessageId, CancellationToken cancellationToken);
+    }
+
+    public sealed class BrokerServer : RunnerService, IBrokerServer
+    {
+        private HttpClient _httpClient;
+
+        public async Task ConnectAsync(Uri serverUrl, CancellationToken cancellationToken)
+        {
+            _httpClient = new HttpClient();
+            _httpClient.BaseAddress = serverUrl;
+            _httpClient.Timeout = TimeSpan.FromSeconds(100);
+            await _httpClient.GetAsync("health", cancellationToken);
+        }
+
+        public async Task<string> GetMessageAsync(TaskAgentSession session, RunnerSettings settings, long? lastMessageId, CancellationToken cancellationToken)
+        {
+            var response = await _httpClient.GetAsync($"message?tenant=org:github&root_tenant=org:github&group_id={settings.PoolId}&group_name={settings.PoolName}&runner_id={settings.AgentId}&runner_name={settings.AgentName}&labels=self-hosted,linux", cancellationToken);
+            if (!response.IsSuccessStatusCode)
+            {
+                var content = default(string);
+                try
+                {
+                    content = await response.Content.ReadAsStringAsync();
+                }
+                catch
+                {
+                }
+
+                var error = $"HTTP {(int)response.StatusCode} {Enum.GetName(typeof(HttpStatusCode), response.StatusCode)}";
+                if (!string.IsNullOrEmpty(content))
+                {
+                    error = $"{error}: {content}";
+                }
+                throw new Exception(error);
+            }
+
+            return await response.Content.ReadAsStringAsync();
+        }
+    }
+}

src/Runner.Common/Constants.cs

@@ -149,6 +149,7 @@ namespace GitHub.Runner.Common
             public static class Features
             {
                 public static readonly string DiskSpaceWarning = "runner.diskspace.warning";
+                public static readonly string Node12Warning = "DistributedTask.AddWarningToNode12Action";
             }
 
             public static readonly string InternalTelemetryIssueDataKey = "_internal_telemetry";
@@ -158,6 +159,7 @@ namespace GitHub.Runner.Common
             public static readonly string UnsupportedCommandMessageDisabled = "The `{0}` command is disabled. Please upgrade to using Environment Files or opt into unsecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_COMMANDS` environment variable to `true`. For more information see: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/";
             public static readonly string UnsupportedStopCommandTokenDisabled = "You cannot use a endToken that is an empty string, the string 'pause-logging', or another workflow command. For more information see: https://docs.github.com/actions/learn-github-actions/workflow-commands-for-github-actions#example-stopping-and-starting-workflow-commands or opt into insecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS` environment variable to `true`.";
             public static readonly string UnsupportedSummarySize = "$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of {0}k, got {1}k. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary";
+            public static readonly string Node12DetectedAfterEndOfLife = "Node.js 12 actions are deprecated. Please update the following actions to use Node.js 16: {0}";
         }
 
         public static class RunnerEvent
@@ -189,6 +191,12 @@ namespace GitHub.Runner.Common
             public static readonly string Success = "success";
         }
 
+        public static class Hooks
+        {
+            public static readonly string JobStartedStepName = "Set up runner";
+            public static readonly string JobCompletedStepName = "Complete runner";
+        }
+
         public static class Path
         {
             public static readonly string ActionsDirectory = "_actions";

src/Runner.Common/JobServer.cs

@@ -2,24 +2,31 @@ using GitHub.DistributedTask.WebApi;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Net.WebSockets;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
 using GitHub.Services.WebApi;
+using GitHub.Services.WebApi.Utilities.Internal;
+using Newtonsoft.Json;
 
 namespace GitHub.Runner.Common
 {
     [ServiceLocator(Default = typeof(JobServer))]
-    public interface IJobServer : IRunnerService
+    public interface IJobServer : IRunnerService, IAsyncDisposable
     {
         Task ConnectAsync(VssConnection jobConnection);
 
+        void InitializeWebsocketClient(ServiceEndpoint serviceEndpoint);
+
         // logging and console
         Task<TaskLog> AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken);
-        Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, CancellationToken cancellationToken);
+        Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, long? startLine, CancellationToken cancellationToken);
-        Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, long startLine, CancellationToken cancellationToken);
         Task<TaskAttachment> CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken);
         Task<TaskLog> CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken);
         Task<Timeline> CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken);
@@ -34,6 +41,20 @@ namespace GitHub.Runner.Common
         private bool _hasConnection;
         private VssConnection _connection;
         private TaskHttpClient _taskClient;
+        private ClientWebSocket _websocketClient;
+
+        private ServiceEndpoint _serviceEndpoint;
+
+        private int totalBatchedLinesAttemptedByWebsocket = 0;
+        private int failedAttemptsToPostBatchedLinesByWebsocket = 0;
+
+
+        private static readonly TimeSpan _minDelayForWebsocketReconnect = TimeSpan.FromMilliseconds(100);
+        private static readonly TimeSpan _maxDelayForWebsocketReconnect = TimeSpan.FromMilliseconds(500);
+        private static readonly int _minWebsocketFailurePercentageAllowed = 50;
+        private static readonly int _minWebsocketBatchedLinesCountToConsider = 5;
+
+        private Task _websocketConnectTask;
 
         public async Task ConnectAsync(VssConnection jobConnection)
         {
@@ -117,6 +138,21 @@ namespace GitHub.Runner.Common
             }
         }
 
+        public void InitializeWebsocketClient(ServiceEndpoint serviceEndpoint)
+        {
+             this._serviceEndpoint = serviceEndpoint;
+             InitializeWebsocketClient(TimeSpan.Zero);
+        }
+
+        public ValueTask DisposeAsync()
+        {
+            CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None);
+
+            GC.SuppressFinalize(this);
+
+            return ValueTask.CompletedTask;
+        }
+
         private void CheckConnection()
         {
             if (!_hasConnection)
@@ -125,6 +161,53 @@ namespace GitHub.Runner.Common
             }
         }
 
+        private void InitializeWebsocketClient(TimeSpan delay)
+        {
+             if (_serviceEndpoint.Authorization != null &&
+                 _serviceEndpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.AccessToken, out var accessToken) &&
+                 !string.IsNullOrEmpty(accessToken))
+            {
+                if (_serviceEndpoint.Data.TryGetValue("FeedStreamUrl", out var feedStreamUrl) && !string.IsNullOrEmpty(feedStreamUrl))
+                {
+                    // let's ensure we use the right scheme
+                    feedStreamUrl = feedStreamUrl.Replace("https://", "wss://").Replace("http://", "ws://");
+                    Trace.Info($"Creating websocket client ..." + feedStreamUrl);
+                    this._websocketClient = new ClientWebSocket();
+                    this._websocketClient.Options.SetRequestHeader("Authorization", $"Bearer {accessToken}");
+                    var userAgentValues = new List<ProductInfoHeaderValue>();
+                    userAgentValues.AddRange(UserAgentUtility.GetDefaultRestUserAgent());
+                    userAgentValues.AddRange(HostContext.UserAgents);
+                    this._websocketClient.Options.SetRequestHeader("User-Agent", string.Join(" ", userAgentValues.Select(x=>x.ToString())));
+
+                    this._websocketConnectTask = ConnectWebSocketClient(feedStreamUrl, delay);
+                }
+                else
+                {
+                    Trace.Info($"No FeedStreamUrl found, so we will use Rest API calls for sending feed data");
+                }
+            }
+            else
+            {
+                Trace.Info($"No access token from the service endpoint");
+            }
+        }
+
+        private async Task ConnectWebSocketClient(string feedStreamUrl, TimeSpan delay)
+        {
+            try
+            {
+                Trace.Info($"Attempting to start websocket client with delay {delay}.");
+                await Task.Delay(delay);
+                await this._websocketClient.ConnectAsync(new Uri(feedStreamUrl), default(CancellationToken));
+                Trace.Info($"Successfully started websocket client.");
+            }
+            catch(Exception ex)
+            {
+                Trace.Info("Exception caught during websocket client connect, fallback of HTTP would be used now instead of websocket.");
+                Trace.Error(ex);
+            }
+        }
+
         //-----------------------------------------------------------------
         // Feedback: WebConsole, TimelineRecords and Logs
         //-----------------------------------------------------------------
@@ -135,16 +218,86 @@ namespace GitHub.Runner.Common
             return _taskClient.AppendLogContentAsync(scopeIdentifier, hubName, planId, logId, uploadStream, cancellationToken: cancellationToken);
         }
 
-        public Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, CancellationToken cancellationToken)
+        public async Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, long? startLine, CancellationToken cancellationToken)
         {
             CheckConnection();
-            return _taskClient.AppendTimelineRecordFeedAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, stepId, lines, cancellationToken: cancellationToken);
+            var pushedLinesViaWebsocket = false;
+            if (_websocketConnectTask != null)
+            {
+                await _websocketConnectTask;
             }
 
-        public Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, long startLine, CancellationToken cancellationToken)
+            // "_websocketClient != null" implies either: We have a successful connection OR we have to attempt sending again and then reconnect
+            // ...in other words, if websocket client is null, we will skip sending to websocket and just use rest api calls to send data
+            if (_websocketClient != null)
             {
-            CheckConnection();
+                var linesWrapper =  startLine.HasValue? new TimelineRecordFeedLinesWrapper(stepId, lines, startLine.Value):  new TimelineRecordFeedLinesWrapper(stepId, lines);
-            return _taskClient.AppendTimelineRecordFeedAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, stepId, lines, startLine, cancellationToken: cancellationToken);
+                var jsonData = StringUtil.ConvertToJson(linesWrapper);
+                try
+                {
+                    totalBatchedLinesAttemptedByWebsocket++;
+                    var jsonDataBytes = Encoding.UTF8.GetBytes(jsonData);
+                    // break the message into chunks of 1024 bytes
+                    for (var i = 0; i < jsonDataBytes.Length; i += 1 * 1024)
+                    {
+                        var lastChunk = i + (1 * 1024) >= jsonDataBytes.Length;
+                        var chunk = new ArraySegment<byte>(jsonDataBytes, i, Math.Min(1 * 1024, jsonDataBytes.Length - i));
+                        await _websocketClient.SendAsync(chunk, WebSocketMessageType.Text, endOfMessage:lastChunk, cancellationToken);
+                    }
+
+                    pushedLinesViaWebsocket = true;
+                }
+                catch (Exception ex)
+                {
+                    failedAttemptsToPostBatchedLinesByWebsocket++;
+                    Trace.Info($"Caught exception during append web console line to websocket, let's fallback to sending via non-websocket call (total calls: {totalBatchedLinesAttemptedByWebsocket}, failed calls: {failedAttemptsToPostBatchedLinesByWebsocket}, websocket state: {this._websocketClient?.State}).");
+                    Trace.Error(ex);
+                    if (totalBatchedLinesAttemptedByWebsocket > _minWebsocketBatchedLinesCountToConsider)
+                    {
+                        // let's consider failure percentage
+                        if (failedAttemptsToPostBatchedLinesByWebsocket * 100 / totalBatchedLinesAttemptedByWebsocket > _minWebsocketFailurePercentageAllowed)
+                        {
+                            Trace.Info($"Exhausted websocket allowed retries, we will not attempt websocket connection for this job to post lines again.");
+                            CloseWebSocket(WebSocketCloseStatus.InternalServerError, cancellationToken);
+
+                            // By setting it to null, we will ensure that we never try websocket path again for this job
+                            _websocketClient = null;
+                        }
+                    }
+
+                    if (_websocketClient != null)
+                    {
+                        var delay = BackoffTimerHelper.GetRandomBackoff(_minDelayForWebsocketReconnect, _maxDelayForWebsocketReconnect);
+                        Trace.Info($"Websocket is not open, let's attempt to connect back again with random backoff {delay} ms (total calls: {totalBatchedLinesAttemptedByWebsocket}, failed calls: {failedAttemptsToPostBatchedLinesByWebsocket}).");
+                        InitializeWebsocketClient(delay);
+                    }
+                }
+            }
+
+            if (!pushedLinesViaWebsocket)
+            {
+                if (startLine.HasValue)
+                {
+                    await _taskClient.AppendTimelineRecordFeedAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, stepId, lines, startLine.Value, cancellationToken: cancellationToken);
+                }
+                else
+                {
+                    await _taskClient.AppendTimelineRecordFeedAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, stepId, lines, cancellationToken: cancellationToken);
+                }
+            }
+        }
+
+        private void CloseWebSocket(WebSocketCloseStatus closeStatus, CancellationToken cancellationToken)
+        {
+            try
+            {
+                _websocketClient?.CloseOutputAsync(closeStatus, "Closing websocket", cancellationToken);
+            }
+            catch (Exception websocketEx)
+            {
+                // In some cases this might be okay since the websocket might be open yet, so just close and don't trace exceptions
+                Trace.Info($"Failed to close websocket gracefully {websocketEx.GetType().Name}");
+            }
         }
 
         public Task<TaskAttachment> CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, string type, string name, Stream uploadStream, CancellationToken cancellationToken)

src/Runner.Common/JobServerQueue.cs

@@ -89,6 +89,10 @@ namespace GitHub.Runner.Common
         {
             Trace.Entering();
 
+            var serviceEndPoint = jobRequest.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
+
+            _jobServer.InitializeWebsocketClient(serviceEndPoint);
+
             if (_queueInProcess)
             {
                 Trace.Info("No-opt, all queue process tasks are running.");
@@ -156,6 +160,9 @@ namespace GitHub.Runner.Common
             await ProcessTimelinesUpdateQueueAsync(runOnce: true);
             Trace.Info("Timeline update queue drained.");
 
+            Trace.Info($"Disposing job server ...");
+            await _jobServer.DisposeAsync();
+
             Trace.Info("All queue process tasks have been stopped, and all queues are drained.");
         }
 
@@ -292,15 +299,7 @@ namespace GitHub.Runner.Common
                         {
                             try
                             {
-                                // we will not requeue failed batch, since the web console lines are time sensitive.
+                                await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), batch[0].LineNumber, default(CancellationToken));
-                                if (batch[0].LineNumber.HasValue)
-                                {
-                                    await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), batch[0].LineNumber.Value, default(CancellationToken));
-                                }
-                                else
-                                {
-                                    await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), default(CancellationToken));
-                                }
 
                                 if (_firstConsoleOutputs)
                                 {

src/Runner.Common/RunServer.cs

@@ -0,0 +1,120 @@
+using GitHub.DistributedTask.WebApi;
+using System;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.Runner.Common.Util;
+using GitHub.Services.WebApi;
+using GitHub.Services.Common;
+using GitHub.Runner.Sdk;
+using GitHub.DistributedTask.Pipelines;
+
+namespace GitHub.Runner.Common
+{
+    [ServiceLocator(Default = typeof(RunServer))]
+    public interface IRunServer : IRunnerService
+    {
+        Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
+
+        Task<AgentJobRequestMessage> GetJobMessageAsync(Guid scopeId, Guid hostId, string planType, string planGroup, Guid planId, IList<InstanceRef> instanceRefsJson);
+    }
+
+    public sealed class RunServer : RunnerService, IRunServer
+    {
+        private bool _hasConnection;
+        private VssConnection _connection;
+        private TaskAgentHttpClient _taskAgentClient;
+
+        public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials)
+        {
+            // System.Console.WriteLine("RunServer.ConnectAsync");
+            _connection = await EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
+            _taskAgentClient = _connection.GetClient<TaskAgentHttpClient>();
+            _hasConnection = true;
+        }
+
+        private async Task<VssConnection> EstablishVssConnection(Uri serverUrl, VssCredentials credentials, TimeSpan timeout)
+        {
+            // System.Console.WriteLine("EstablishVssConnection");
+            Trace.Info($"EstablishVssConnection");
+            Trace.Info($"Establish connection with {timeout.TotalSeconds} seconds timeout.");
+            int attemptCount = 5;
+            while (attemptCount-- > 0)
+            {
+                var connection = VssUtil.CreateConnection(serverUrl, credentials, timeout: timeout);
+                try
+                {
+                    await connection.ConnectAsync();
+                    return connection;
+                }
+                catch (Exception ex) when (attemptCount > 0)
+                {
+                    Trace.Info($"Catch exception during connect. {attemptCount} attempt left.");
+                    Trace.Error(ex);
+
+                    await HostContext.Delay(TimeSpan.FromMilliseconds(100), CancellationToken.None);
+                }
+            }
+
+            // should never reach here.
+            throw new InvalidOperationException(nameof(EstablishVssConnection));
+        }
+
+        private void CheckConnection()
+        {
+            if (!_hasConnection)
+            {
+                throw new InvalidOperationException($"SetConnection");
+            }
+        }
+
+        public Task<AgentJobRequestMessage> GetJobMessageAsync(
+            Guid scopeId,
+            Guid hostId,
+            string planType,
+            string planGroup,
+            Guid planId,
+            IList<InstanceRef> instanceRefsJson)
+        {
+            // System.Console.WriteLine("RunServer.GetMessageAsync");
+            CheckConnection();
+            return _taskAgentClient.GetJobMessageAsync(scopeId, hostId, planType, planGroup, planId, StringUtil.ConvertToJson(instanceRefsJson, Newtonsoft.Json.Formatting.None));
+        }
+    }
+
+    // todo: move to SDK?
+    [DataContract]
+    public sealed class MessageRef
+    {
+        [DataMember(Name = "url")]
+        public string Url { get; set; }
+        [DataMember(Name = "token")]
+        public string Token { get; set; }
+        [DataMember(Name = "scopeId")]
+        public Guid ScopeId { get; set; }
+        [DataMember(Name = "hostId")]
+        public Guid HostId { get; set; }
+        [DataMember(Name = "planType")]
+        public string PlanType { get; set; }
+        [DataMember(Name = "planGroup")]
+        public string PlanGroup { get; set; }
+        [DataMember(Name = "planId")]
+        public Guid PlanId { get; set; }
+        [DataMember(Name = "instanceRefs")]
+        public InstanceRef[] InstanceRefs { get; set; }
+        [DataMember(Name = "labels")]
+        public string[] Labels { get; set; }
+    }
+
+    [DataContract]
+    public sealed class InstanceRef
+    {
+        [DataMember(Name = "name")]
+        public string Name { get; set; }
+        [DataMember(Name = "instanceType")]
+        public string InstanceType { get; set; }
+        [DataMember(Name = "attempt")]
+        public int Attempt { get; set; }
+    }
+}

src/Runner.Common/RunnerServer.cs

@@ -44,7 +44,7 @@ namespace GitHub.Runner.Common
         // job request
         Task<TaskAgentJobRequest> GetAgentRequestAsync(int poolId, long requestId, CancellationToken cancellationToken);
         Task<TaskAgentJobRequest> RenewAgentRequestAsync(int poolId, long requestId, Guid lockToken, string orchestrationId, CancellationToken cancellationToken);
-        Task<TaskAgentJobRequest> FinishAgentRequestAsync(int poolId, long requestId, Guid lockToken, DateTime finishTime, TaskResult result, CancellationToken cancellationToken);
+        Task<TaskAgentJobRequest> FinishAgentRequestAsync(int poolId, long requestId, Guid lockToken, DateTime finishTime, TaskResult result, Guid targetHostId, CancellationToken cancellationToken);
 
         // agent package
         Task<List<PackageMetadata>> GetPackagesAsync(string packageType, string platform, int top, bool includeToken, CancellationToken cancellationToken);
@@ -68,11 +68,23 @@ namespace GitHub.Runner.Common
 
         public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials)
         {
-            var createGenericConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
+            // System.Console.WriteLine("RunnerServer.ConnectAsync: Create message connection");
             var createMessageConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
-            var createRequestConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
+            await Task.WhenAll(createMessageConnection);
 
-            await Task.WhenAll(createGenericConnection, createMessageConnection, createRequestConnection);
+            // System.Console.WriteLine("RunnerServer.ConnectAsync: Create generic connection");
+            var createGenericConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
+            await Task.WhenAll(createGenericConnection);
+
+            // System.Console.WriteLine("RunnerServer.ConnectAsync: Create request connection");
+            var createRequestConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
+            await Task.WhenAll(createRequestConnection);
+
+            // var createGenericConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
+            // var createMessageConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
+            // var createRequestConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
+
+            // await Task.WhenAll(createGenericConnection, createMessageConnection, createRequestConnection);
 
             _genericConnection = await createGenericConnection;
             _messageConnection = await createMessageConnection;
@@ -182,6 +194,8 @@ namespace GitHub.Runner.Common
 
         private async Task<VssConnection> EstablishVssConnection(Uri serverUrl, VssCredentials credentials, TimeSpan timeout)
         {
+            // System.Console.WriteLine("EstablishVssConnection");
+            Trace.Info($"EstablishVssConnection");
             Trace.Info($"Establish connection with {timeout.TotalSeconds} seconds timeout.");
             int attemptCount = 5;
             while (attemptCount-- > 0)
@@ -238,41 +252,48 @@ namespace GitHub.Runner.Common
 
         public Task<List<TaskAgentPool>> GetAgentPoolsAsync(string agentPoolName = null, TaskAgentPoolType poolType = TaskAgentPoolType.Automation)
         {
+            // System.Console.WriteLine("RunnerServer.GetAgentPoolsAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.GetAgentPoolsAsync(agentPoolName, poolType: poolType);
         }
 
         public Task<TaskAgent> AddAgentAsync(Int32 agentPoolId, TaskAgent agent)
         {
+            // System.Console.WriteLine("RunnerServer.AddAgentAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.AddAgentAsync(agentPoolId, agent);
         }
 
         public Task<List<TaskAgent>> GetAgentsAsync(int agentPoolId, string agentName = null)
         {
+            // System.Console.WriteLine("RunnerServer.GetAgentsAsync 1");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.GetAgentsAsync(agentPoolId, agentName, false);
         }
 
         public Task<List<TaskAgent>> GetAgentsAsync(string agentName)
         {
+            // System.Console.WriteLine("RunnerServer.GetAgentsAsync 2");
             return GetAgentsAsync(0, agentName); // search in all all agentPools
         }
 
         public Task<TaskAgent> ReplaceAgentAsync(int agentPoolId, TaskAgent agent)
         {
+            // System.Console.WriteLine("RunnerServer.ReplaceAgentAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.ReplaceAgentAsync(agentPoolId, agent);
         }
 
         public Task DeleteAgentAsync(int agentPoolId, int agentId)
         {
+            // System.Console.WriteLine("RunnerServer.DeleteAgentAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.DeleteAgentAsync(agentPoolId, agentId);
         }
 
         public Task DeleteAgentAsync(int agentId)
         {
+            // System.Console.WriteLine("RunnerServer.DeleteAgentAsync");
             return DeleteAgentAsync(0, agentId); // agentPool is ignored server side
         }
 
@@ -282,24 +303,28 @@ namespace GitHub.Runner.Common
 
         public Task<TaskAgentSession> CreateAgentSessionAsync(Int32 poolId, TaskAgentSession session, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("RunnerServer.CreateAgentSessionAsync");
             CheckConnection(RunnerConnectionType.MessageQueue);
             return _messageTaskAgentClient.CreateAgentSessionAsync(poolId, session, cancellationToken: cancellationToken);
         }
 
         public Task DeleteAgentMessageAsync(Int32 poolId, Int64 messageId, Guid sessionId, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("RunnerServer.DeleteAgentMessageAsync");
             CheckConnection(RunnerConnectionType.MessageQueue);
             return _messageTaskAgentClient.DeleteMessageAsync(poolId, messageId, sessionId, cancellationToken: cancellationToken);
         }
 
         public Task DeleteAgentSessionAsync(Int32 poolId, Guid sessionId, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("RunnerServer.DeleteAgentSessionAsync");
             CheckConnection(RunnerConnectionType.MessageQueue);
             return _messageTaskAgentClient.DeleteAgentSessionAsync(poolId, sessionId, cancellationToken: cancellationToken);
         }
 
         public Task<TaskAgentMessage> GetAgentMessageAsync(Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("RunnerServer.GetAgentMessageAsync");
             CheckConnection(RunnerConnectionType.MessageQueue);
             return _messageTaskAgentClient.GetMessageAsync(poolId, sessionId, lastMessageId, cancellationToken: cancellationToken);
         }
@@ -310,18 +335,21 @@ namespace GitHub.Runner.Common
 
         public Task<TaskAgentJobRequest> RenewAgentRequestAsync(int poolId, long requestId, Guid lockToken, string orchestrationId = null, CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("RunnerServer.RenewAgentRequestAsync");
             CheckConnection(RunnerConnectionType.JobRequest);
             return _requestTaskAgentClient.RenewAgentRequestAsync(poolId, requestId, lockToken, orchestrationId: orchestrationId, cancellationToken: cancellationToken);
         }
 
-        public Task<TaskAgentJobRequest> FinishAgentRequestAsync(int poolId, long requestId, Guid lockToken, DateTime finishTime, TaskResult result, CancellationToken cancellationToken = default(CancellationToken))
+        public Task<TaskAgentJobRequest> FinishAgentRequestAsync(int poolId, long requestId, Guid lockToken, DateTime finishTime, TaskResult result, Guid targetHostId, CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("RunnerServer.FinishAgentRequestAsync");
             CheckConnection(RunnerConnectionType.JobRequest);
-            return _requestTaskAgentClient.FinishAgentRequestAsync(poolId, requestId, lockToken, finishTime, result, cancellationToken: cancellationToken);
+            return _requestTaskAgentClient.FinishAgentRequestAsync(poolId, requestId, lockToken, finishTime, result, targetHostId, cancellationToken: cancellationToken);
         }
 
         public Task<TaskAgentJobRequest> GetAgentRequestAsync(int poolId, long requestId, CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("RunnerServer.GetAgentRequestAsync");
             CheckConnection(RunnerConnectionType.JobRequest);
             return _requestTaskAgentClient.GetAgentRequestAsync(poolId, requestId, cancellationToken: cancellationToken);
         }
@@ -331,18 +359,21 @@ namespace GitHub.Runner.Common
         //-----------------------------------------------------------------
         public Task<List<PackageMetadata>> GetPackagesAsync(string packageType, string platform, int top, bool includeToken, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("RunnerServer.GetPackagesAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.GetPackagesAsync(packageType, platform, top, includeToken, cancellationToken: cancellationToken);
         }
 
         public Task<PackageMetadata> GetPackageAsync(string packageType, string platform, string version, bool includeToken, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("RunnerServer.GetPackageAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.GetPackageAsync(packageType, platform, version, includeToken, cancellationToken: cancellationToken);
         }
 
         public Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, int agentId, string currentState, string trace)
         {
+            // System.Console.WriteLine("RunnerServer.UpdateAgentUpdateStateAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.UpdateAgentUpdateStateAsync(agentPoolId, agentId, currentState, trace);
         }

src/Runner.Listener/Configuration/ConfigurationManager.cs

@@ -624,9 +624,12 @@ namespace GitHub.Runner.Listener.Configuration
                     httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("basic", base64EncodingToken);
                     httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
                     httpClient.DefaultRequestHeaders.Accept.ParseAdd("application/vnd.github.v3+json");
+                    
+                    var responseStatus = System.Net.HttpStatusCode.OK;
                     try
                     {
                         var response = await httpClient.PostAsync(githubApiUrl, new StringContent(string.Empty));
+                        responseStatus = response.StatusCode;
 
                         if (response.IsSuccessStatusCode)
                         {
@@ -634,11 +637,6 @@ namespace GitHub.Runner.Listener.Configuration
                             var jsonResponse = await response.Content.ReadAsStringAsync();
                             return StringUtil.ConvertFromJson<GitHubRunnerRegisterToken>(jsonResponse);
                         }
-                        else if(response.StatusCode == System.Net.HttpStatusCode.NotFound)
-                        {
-                            // It doesn't make sense to retry in this case, so just stop
-                            break;
-                        }
                         else
                         {
                             _term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
@@ -647,15 +645,15 @@ namespace GitHub.Runner.Listener.Configuration
                             response.EnsureSuccessStatusCode();
                         }
                     }
-                    catch(Exception ex) when (retryCount < 2)
+                    catch(Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound)
                     {
                         retryCount++;
                         Trace.Error($"Failed to get JIT runner token -- Atempt: {retryCount}");
                         Trace.Error(ex);
-                        Trace.Info("Retrying in 5 seconds");
                     }
                 }
                 var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5));
+                Trace.Info($"Retrying in {backOff.Seconds} seconds");
                 await Task.Delay(backOff);
             }
             return null;
@@ -689,9 +687,11 @@ namespace GitHub.Runner.Listener.Configuration
                         {"runner_event", runnerEvent}
                     };
 
+                    var responseStatus = System.Net.HttpStatusCode.OK;
                     try
                     {
                         var response = await httpClient.PostAsync(githubApiUrl, new StringContent(StringUtil.ConvertToJson(bodyObject), null, "application/json"));
+                        responseStatus = response.StatusCode;
 
                         if(response.IsSuccessStatusCode)
                         {
@@ -699,29 +699,23 @@ namespace GitHub.Runner.Listener.Configuration
                             var jsonResponse = await response.Content.ReadAsStringAsync();
                             return StringUtil.ConvertFromJson<GitHubAuthResult>(jsonResponse);
                         }
-                        else if(response.StatusCode == System.Net.HttpStatusCode.NotFound)
-                        {
-                            // It doesn't make sense to retry in this case, so just stop
-                            break;
-                        }
                         else
                         {
                             _term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
                             var errorResponse = await response.Content.ReadAsStringAsync();
                             _term.WriteError(errorResponse);
-                            // Something else bad happened, let's go to our retry logic
                             response.EnsureSuccessStatusCode();
                         }
                     }
-                    catch(Exception ex) when (retryCount < 2)
+                    catch(Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound)
                     {
                         retryCount++;
                         Trace.Error($"Failed to get tenant credentials -- Atempt: {retryCount}");
                         Trace.Error(ex);
-                        Trace.Info("Retrying in 5 seconds");
                     }
                 }
                 var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5));
+                Trace.Info($"Retrying in {backOff.Seconds} seconds");
                 await Task.Delay(backOff);
             }
             return null;

src/Runner.Listener/Configuration/CredentialManager.cs

@@ -40,6 +40,12 @@ namespace GitHub.Runner.Listener.Configuration
             return creds;
         }
 
+#if USE_BROKER
+        public VssCredentials LoadCredentials()
+        {
+            return new VssCredentials();
+        }
+#else
         public VssCredentials LoadCredentials()
         {
             IConfigurationStore store = HostContext.GetService<IConfigurationStore>();
@@ -69,6 +75,7 @@ namespace GitHub.Runner.Listener.Configuration
 
             return creds;
         }
+#endif
     }
 
     [DataContract]

src/Runner.Listener/JobDispatcher.cs

@@ -23,7 +23,7 @@ namespace GitHub.Runner.Listener
     {
         bool Busy { get; }
         TaskCompletionSource<bool> RunOnceJobCompleted { get; }
-        void Run(Pipelines.AgentJobRequestMessage message, bool runOnce = false);
+        void Run(Guid targetHostId, Pipelines.AgentJobRequestMessage message, bool runOnce = false);
         bool Cancel(JobCancelMessage message);
         Task WaitAsync(CancellationToken token);
         Task ShutdownAsync();
@@ -79,7 +79,7 @@ namespace GitHub.Runner.Listener
 
         public bool Busy { get; private set; }
 
-        public void Run(Pipelines.AgentJobRequestMessage jobRequestMessage, bool runOnce = false)
+        public void Run(Guid targetHostId, Pipelines.AgentJobRequestMessage jobRequestMessage, bool runOnce = false)
         {
             Trace.Info($"Job request {jobRequestMessage.RequestId} for plan {jobRequestMessage.Plan.PlanId} job {jobRequestMessage.JobId} received.");
 
@@ -112,11 +112,11 @@ namespace GitHub.Runner.Listener
             if (runOnce)
             {
                 Trace.Info("Start dispatcher for one time used runner.");
-                newDispatch.WorkerDispatch = RunOnceAsync(jobRequestMessage, orchestrationId, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token, newDispatch.WorkerCancelTimeoutKillTokenSource.Token);
+                newDispatch.WorkerDispatch = RunOnceAsync(targetHostId, jobRequestMessage, orchestrationId, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token, newDispatch.WorkerCancelTimeoutKillTokenSource.Token);
             }
             else
             {
-                newDispatch.WorkerDispatch = RunAsync(jobRequestMessage, orchestrationId, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token, newDispatch.WorkerCancelTimeoutKillTokenSource.Token);
+                newDispatch.WorkerDispatch = RunAsync(targetHostId, jobRequestMessage, orchestrationId, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token, newDispatch.WorkerCancelTimeoutKillTokenSource.Token);
             }
 
             _jobInfos.TryAdd(newDispatch.JobId, newDispatch);
@@ -317,11 +317,11 @@ namespace GitHub.Runner.Listener
             }
         }
 
-        private async Task RunOnceAsync(Pipelines.AgentJobRequestMessage message, string orchestrationId, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
+        private async Task RunOnceAsync(Guid targetHostId, Pipelines.AgentJobRequestMessage message, string orchestrationId, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
         {
             try
             {
-                await RunAsync(message, orchestrationId, previousJobDispatch, jobRequestCancellationToken, workerCancelTimeoutKillToken);
+                await RunAsync(targetHostId, message, orchestrationId, previousJobDispatch, jobRequestCancellationToken, workerCancelTimeoutKillToken);
             }
             finally
             {
@@ -330,7 +330,7 @@ namespace GitHub.Runner.Listener
             }
         }
 
-        private async Task RunAsync(Pipelines.AgentJobRequestMessage message, string orchestrationId, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
+        private async Task RunAsync(Guid targetHostId, Pipelines.AgentJobRequestMessage message, string orchestrationId, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
         {
             Busy = true;
             try
@@ -383,7 +383,7 @@ namespace GitHub.Runner.Listener
                         await renewJobRequest;
 
                         // complete job request with result Cancelled
-                        await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
+                        await CompleteJobRequestAsync(targetHostId, _poolId, message, lockToken, TaskResult.Canceled);
                         return;
                     }
 
@@ -540,7 +540,7 @@ namespace GitHub.Runner.Listener
                                 await renewJobRequest;
 
                                 // complete job request
-                                await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo);
+                                await CompleteJobRequestAsync(targetHostId, _poolId, message, lockToken, result, detailInfo);
 
                                 // print out unhandled exception happened in worker after we complete job request.
                                 // when we run out of disk space, report back to server has higher priority.
@@ -637,7 +637,7 @@ namespace GitHub.Runner.Listener
                             await renewJobRequest;
 
                             // complete job request
-                            await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
+                            await CompleteJobRequestAsync(targetHostId, _poolId, message, lockToken, resultOnAbandonOrCancel);
                         }
                         finally
                         {
@@ -666,17 +666,25 @@ namespace GitHub.Runner.Listener
             {
                 try
                 {
-                    request = await runnerServer.RenewAgentRequestAsync(poolId, requestId, lockToken, orchestrationId, token);
+// #if USE_BROKER
-                    Trace.Info($"Successfully renew job request {requestId}, job is valid till {request.LockedUntil.Value}");
-
                     if (!firstJobRequestRenewed.Task.IsCompleted)
                     {
                         // fire first renew succeed event.
                         firstJobRequestRenewed.TrySetResult(0);
-
-                        // Update settings if the runner name has been changed server-side
-                        UpdateAgentNameIfNeeded(request.ReservedAgent?.Name);
                     }
+// #else
+//                     request = await runnerServer.RenewAgentRequestAsync(poolId, requestId, lockToken, orchestrationId, token);
+//                     Trace.Info($"Successfully renew job request {requestId}, job is valid till {request?.LockedUntil.Value}");
+
+//                     if (!firstJobRequestRenewed.Task.IsCompleted)
+//                     {
+//                         // fire first renew succeed event.
+//                         firstJobRequestRenewed.TrySetResult(0);
+
+//                         // Update settings if the runner name has been changed server-side
+//                         UpdateAgentNameIfNeeded(request.ReservedAgent?.Name);
+//                     }
+// #endif
 
                     if (encounteringError > 0)
                     {
@@ -911,7 +919,7 @@ namespace GitHub.Runner.Listener
             }
         }
 
-        private async Task CompleteJobRequestAsync(int poolId, Pipelines.AgentJobRequestMessage message, Guid lockToken, TaskResult result, string detailInfo = null)
+        private async Task CompleteJobRequestAsync(Guid targetHostId, int poolId, Pipelines.AgentJobRequestMessage message, Guid lockToken, TaskResult result, string detailInfo = null)
         {
             Trace.Entering();
 
@@ -928,7 +936,7 @@ namespace GitHub.Runner.Listener
             {
                 try
                 {
-                    await runnerServer.FinishAgentRequestAsync(poolId, message.RequestId, lockToken, DateTime.UtcNow, result, CancellationToken.None);
+                    await runnerServer.FinishAgentRequestAsync(poolId, message.RequestId, lockToken, DateTime.UtcNow, result, targetHostId, CancellationToken.None);
                     return;
                 }
                 catch (TaskAgentJobNotFoundException)

src/Runner.Listener/MessageListener.cs

@@ -2,7 +2,11 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
 using System.Runtime.InteropServices;
+using System.Runtime.Serialization;
 using System.Security.Cryptography;
 using System.Text;
 using System.Threading;
@@ -27,10 +31,13 @@ namespace GitHub.Runner.Listener
 
     public sealed class MessageListener : RunnerService, IMessageListener
     {
+#if !USE_BROKER
         private long? _lastMessageId;
+#endif
         private RunnerSettings _settings;
         private ITerminal _term;
         private IRunnerServer _runnerServer;
+        private IBrokerServer _brokerServer;
         private TaskAgentSession _session;
         private TimeSpan _getNextMessageRetryInterval;
         private bool _accessTokenRevoked = false;
@@ -45,8 +52,44 @@ namespace GitHub.Runner.Listener
 
             _term = HostContext.GetService<ITerminal>();
             _runnerServer = HostContext.GetService<IRunnerServer>();
+            _brokerServer = HostContext.GetService<IBrokerServer>();
         }
 
+#if USE_BROKER
+        public async Task<Boolean> CreateSessionAsync(CancellationToken token)
+        {
+            Trace.Entering();
+
+            // Settings
+            var configManager = HostContext.GetService<IConfigurationManager>();
+            _settings = configManager.LoadSettings();
+            var serverUrl = _settings.ServerUrl;
+            Trace.Info(_settings);
+
+            // Connect
+            token.ThrowIfCancellationRequested();
+            Trace.Info($"Attempt to create session.");
+            Trace.Info("Connecting to the Runner Server...");
+            _term.WriteLine($"Connecting to {new Uri(serverUrl)}");
+            await _brokerServer.ConnectAsync(new Uri(serverUrl), token);
+            _term.WriteLine();
+            _term.WriteSuccessMessage("Connected to GitHub");
+            _term.WriteLine();
+
+            // Session info
+            var agent = new TaskAgentReference
+            {
+                Id = _settings.AgentId,
+                Name = _settings.AgentName,
+                Version = BuildConstants.RunnerPackage.Version,
+                OSDescription = RuntimeInformation.OSDescription,
+            };
+            string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
+            _session = new TaskAgentSession(sessionName, agent);
+
+            return true;
+        }
+#else
         public async Task<Boolean> CreateSessionAsync(CancellationToken token)
         {
             Trace.Entering();
@@ -81,6 +124,7 @@ namespace GitHub.Runner.Listener
                 Trace.Info($"Attempt to create session.");
                 try
                 {
+                    Trace.Info("Connecting to the Runner Server...");
                     Trace.Info("Connecting to the Runner Server...");
                     await _runnerServer.ConnectAsync(new Uri(serverUrl), creds);
                     Trace.Info("VssConnection created");
@@ -151,6 +195,7 @@ namespace GitHub.Runner.Listener
                 }
             }
         }
+#endif
 
         public async Task DeleteSessionAsync()
         {
@@ -170,6 +215,167 @@ namespace GitHub.Runner.Listener
             }
         }
 
+#if USE_BROKER
+        [DataContract]
+        public sealed class MessageRef
+        {
+            [DataMember(Name = "url")]
+            public string Url { get; set; }
+            [DataMember(Name = "token")]
+            public string Token { get; set; }
+            [DataMember(Name = "scopeId")]
+            public string ScopeId { get; set; }
+            [DataMember(Name = "planType")]
+            public string PlanType { get; set; }
+            [DataMember(Name = "planGroup")]
+            public string PlanGroup { get; set; }
+            [DataMember(Name = "instanceRefs")]
+            public InstanceRef[] InstanceRefs { get; set; }
+            [DataMember(Name = "labels")]
+            public string[] Labels { get; set; }
+        }
+
+        [DataContract]
+        public sealed class InstanceRef
+        {
+            [DataMember(Name = "name")]
+            public string Name { get; set; }
+            [DataMember(Name = "instanceType")]
+            public string InstanceType { get; set; }
+            [DataMember(Name = "attempt")]
+            public int Attempt { get; set; }
+        }
+
+        public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
+        {
+            Trace.Entering();
+            ArgUtil.NotNull(_session, nameof(_session));
+            ArgUtil.NotNull(_settings, nameof(_settings));
+            bool encounteringError = false;
+            int continuousError = 0;
+            string errorMessage = string.Empty;
+            Stopwatch heartbeat = new Stopwatch();
+            heartbeat.Restart();
+            while (true)
+            {
+                token.ThrowIfCancellationRequested();
+                string message = null;
+                try
+                {
+                    message = await _brokerServer.GetMessageAsync(_session, _settings, null/*_lastMessageId*/, token);
+                    _term.WriteLine($"{DateTime.UtcNow:u}: {message}");
+                    if (!string.IsNullOrEmpty(message))
+                    {
+                        var messageRef = StringUtil.ConvertFromJson<MessageRef>(message);
+                        var client = new HttpClient();
+                        client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", messageRef.Token);
+                        var response = await client.GetAsync(messageRef.Url, token);
+                        if (!response.IsSuccessStatusCode)
+                        {
+                            var content = default(string);
+                            try
+                            {
+                                content = await response.Content.ReadAsStringAsync();
+                            }
+                            catch
+                            {
+                            }
+
+                            var error = $"HTTP {(int)response.StatusCode} {Enum.GetName(typeof(HttpStatusCode), response.StatusCode)}";
+                            if (!string.IsNullOrEmpty(content))
+                            {
+                                error = $"{error}: {content}";
+                            }
+                            throw new Exception(error);
+                        }
+
+                        var fullMessage = await response.Content.ReadAsStringAsync();
+                        return StringUtil.ConvertFromJson<TaskAgentMessage>(fullMessage);
+                    }
+
+                    if (encounteringError) //print the message once only if there was an error
+                    {
+                        _term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
+                        encounteringError = false;
+                        continuousError = 0;
+                    }
+                }
+                catch (OperationCanceledException) when (token.IsCancellationRequested)
+                {
+                    Trace.Info("Get next message has been cancelled.");
+                    throw;
+                }
+                catch (TaskAgentAccessTokenExpiredException)
+                {
+                    Trace.Info("Runner OAuth token has been revoked. Unable to pull message.");
+                    _accessTokenRevoked = true;
+                    throw;
+                }
+                catch (Exception ex)
+                {
+                    Trace.Error("Catch exception during get next message.");
+                    Trace.Error(ex);
+
+                    // don't retry if SkipSessionRecover = true, DT service will delete agent session to stop agent from taking more jobs.
+                    if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && await CreateSessionAsync(token))
+                    {
+                        Trace.Info($"{nameof(TaskAgentSessionExpiredException)} received, recovered by recreate session.");
+                    }
+                    else if (!IsGetNextMessageExceptionRetriable(ex))
+                    {
+                        throw;
+                    }
+                    else
+                    {
+                        continuousError++;
+                        //retry after a random backoff to avoid service throttling
+                        //in case of there is a service error happened and all agents get kicked off of the long poll and all agent try to reconnect back at the same time.
+                        if (continuousError <= 5)
+                        {
+                            // random backoff [15, 30]
+                            _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval);
+                        }
+                        else
+                        {
+                            // more aggressive backoff [30, 60]
+                            _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(60), _getNextMessageRetryInterval);
+                        }
+
+                        if (!encounteringError)
+                        {
+                            //print error only on the first consecutive error
+                            _term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
+                            encounteringError = true;
+                        }
+
+                        // re-create VssConnection before next retry
+                        await _runnerServer.RefreshConnectionAsync(RunnerConnectionType.MessageQueue, TimeSpan.FromSeconds(60));
+
+                        Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
+                        await HostContext.Delay(_getNextMessageRetryInterval, token);
+                    }
+                }
+
+                // if (message == null)
+                // {
+                //     if (heartbeat.Elapsed > TimeSpan.FromMinutes(30))
+                //     {
+                //         Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes.");
+                //         heartbeat.Restart();
+                //     }
+                //     else
+                //     {
+                //         Trace.Verbose($"No message retrieved from session '{_session.SessionId}'.");
+                //     }
+
+                //     continue;
+                // }
+
+                // Trace.Info($"Message '{message.MessageId}' received from session '{_session.SessionId}'.");
+                // return message;
+            }
+        }
+#else
         public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
         {
             Trace.Entering();
@@ -281,6 +487,7 @@ namespace GitHub.Runner.Listener
                 return message;
             }
         }
+#endif
 
         public async Task DeleteMessageAsync(TaskAgentMessage message)
         {

src/Runner.Listener/Runner.cs

@@ -13,6 +13,10 @@ using GitHub.Runner.Sdk;
 using System.Linq;
 using GitHub.Runner.Listener.Check;
 using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
 
 namespace GitHub.Runner.Listener
 {
@@ -449,7 +453,37 @@ namespace GitHub.Runner.Listener
                                 {
                                     Trace.Info($"Received job message of length {message.Body.Length} from service, with hash '{IOUtil.GetSha256Hash(message.Body)}'");
                                     var jobMessage = StringUtil.ConvertFromJson<Pipelines.AgentJobRequestMessage>(message.Body);
-                                    jobDispatcher.Run(jobMessage, runOnce);
+                                    jobDispatcher.Run(Guid.Empty, jobMessage, runOnce);
+                                    if (runOnce)
+                                    {
+                                        Trace.Info("One time used runner received job message.");
+                                        runOnceJobReceived = true;
+                                    }
+                                }
+                            }
+                            // Broker flow
+                            else if (string.Equals(message.MessageType, JobRequestMessageTypes.RunnerJobRequest, StringComparison.OrdinalIgnoreCase))
+                            {
+                                if (autoUpdateInProgress || runOnceJobReceived)
+                                {
+                                    skipMessageDeletion = true;
+                                    Trace.Info($"Skip message deletion for job request message '{message.MessageId}'.");
+                                }
+                                else
+                                {
+                                    var messageRef = StringUtil.ConvertFromJson<MessageRef>(message.Body);
+
+                                    // Create connection
+                                    var credMgr = HostContext.GetService<ICredentialManager>();
+                                    var creds = credMgr.LoadCredentials();
+
+                                    // todo: add retries
+                                    var runServer = HostContext.CreateService<IRunServer>();
+                                    await runServer.ConnectAsync(new Uri(settings.ServerUrl), creds);
+                                    var jobMessage = await runServer.GetJobMessageAsync(messageRef.ScopeId, messageRef.HostId, messageRef.PlanType, messageRef.PlanGroup, messageRef.PlanId, messageRef.InstanceRefs);
+
+                                    // todo: Trace.Info($"Received job message of length {message.Body.Length} from service, with hash '{IOUtil.GetSha256Hash(message.Body)}'");
+                                    jobDispatcher.Run(messageRef.HostId, jobMessage, runOnce);
                                     if (runOnce)
                                     {
                                         Trace.Info("One time used runner received job message.");

src/Runner.Sdk/Util/VssUtil.cs

@@ -36,6 +36,7 @@ namespace GitHub.Runner.Sdk
 
         public static VssConnection CreateConnection(Uri serverUri, VssCredentials credentials, IEnumerable<DelegatingHandler> additionalDelegatingHandler = null, TimeSpan? timeout = null)
         {
+            // System.Console.WriteLine("VssUtil.CreateConnection");
             VssClientHttpRequestSettings settings = VssClientHttpRequestSettings.Default.Clone();
 
             int maxRetryRequest;

src/Runner.Worker/ActionManager.cs

@@ -817,6 +817,7 @@ namespace GitHub.Runner.Worker
                                     {
                                         // Something else bad happened, let's go to our retry logic
                                         response.EnsureSuccessStatusCode();
+                                        throw new Exception("Unexpected response code: " + response.StatusCode);
                                     }
                                 }
                             }

src/Runner.Worker/ActionRunner.cs

@@ -1,7 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.IO;
-using System.Text;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.ObjectTemplating;
 using GitHub.DistributedTask.ObjectTemplating.Tokens;
@@ -11,6 +9,7 @@ using GitHub.DistributedTask.Pipelines.ObjectTemplating;
 using GitHub.Runner.Common;
 using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
+using GitHub.Runner.Worker;
 using GitHub.Runner.Worker.Handlers;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
@@ -141,21 +140,7 @@ namespace GitHub.Runner.Worker
 
             IStepHost stepHost = HostContext.CreateService<IDefaultStepHost>();
 
-            // Makes directory for event_path data
+            ExecutionContext.WriteWebhookPayload();
-            var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);
-            var workflowDirectory = Path.Combine(tempDirectory, "_github_workflow");
-            Directory.CreateDirectory(workflowDirectory);
-
-            var gitHubEvent = ExecutionContext.GetGitHubContext("event");
-
-            // adds the GitHub event path/file if the event exists
-            if (gitHubEvent != null)
-            {
-                var workflowFile = Path.Combine(workflowDirectory, "event.json");
-                Trace.Info($"Write event payload to {workflowFile}");
-                File.WriteAllText(workflowFile, gitHubEvent, new UTF8Encoding(false));
-                ExecutionContext.SetGitHubContext("event_path", workflowFile);
-            }
 
             // Set GITHUB_ACTION_REPOSITORY if this Action is from a repository
             if (Action.Reference is Pipelines.RepositoryPathReference repoPathReferenceAction &&

src/Runner.Worker/ExecutionContext.cs

@@ -11,6 +11,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using System.Web;
 using GitHub.DistributedTask.Expressions2;
+using GitHub.DistributedTask.ObjectTemplating.Tokens;
 using GitHub.DistributedTask.Pipelines;
 using GitHub.DistributedTask.Pipelines.ContextData;
 using GitHub.DistributedTask.Pipelines.ObjectTemplating;
@@ -109,6 +110,12 @@ namespace GitHub.Runner.Worker
         void ForceTaskComplete();
         void RegisterPostJobStep(IStep step);
         void PublishStepTelemetry();
+
+        void ApplyContinueOnError(TemplateToken continueOnError);
+        void UpdateGlobalStepsContext();
+
+        void WriteWebhookPayload();
+
     }
 
     public sealed class ExecutionContext : RunnerService, IExecutionContext
@@ -438,14 +445,19 @@ namespace GitHub.Runner.Worker
 
             _logger.End();
 
+            UpdateGlobalStepsContext();
+
+            return Result.Value;
+        }
+
+        public void UpdateGlobalStepsContext()
+        {
             // Skip if generated context name. Generated context names start with "__". After 3.2 the server will never send an empty context name.
             if (!string.IsNullOrEmpty(ContextName) && !ContextName.StartsWith("__", StringComparison.Ordinal))
             {
                 Global.StepsContext.SetOutcome(ScopeName, ContextName, (Outcome ?? Result ?? TaskResult.Succeeded).ToActionResult());
                 Global.StepsContext.SetConclusion(ScopeName, ContextName, (Result ?? TaskResult.Succeeded).ToActionResult());
             }
-
-            return Result.Value;
         }
 
         public void SetRunnerContext(string name, string value)
@@ -550,10 +562,15 @@ namespace GitHub.Runner.Worker
                 issue.Message = issue.Message[.._maxIssueMessageLength];
             }
 
+            // Tracking the line number (logFileLineNumber) and step number (stepNumber) for each issue that gets created
+            // Actions UI from the run summary page use both values to easily link to an exact locations in logs where annotations originate from
+            if (_record.Order != null)
+            {
+                issue.Data["stepNumber"] = _record.Order.ToString();
+            }
+
             if (issue.Type == IssueType.Error)
             {
-                // tracking line number for each issue in log file
-                // log UI use this to navigate from issue to log
                 if (!string.IsNullOrEmpty(logMessage))
                 {
                     long logLineNumber = Write(WellKnownTags.Error, logMessage);
@@ -569,8 +586,6 @@ namespace GitHub.Runner.Worker
             }
             else if (issue.Type == IssueType.Warning)
             {
-                // tracking line number for each issue in log file
-                // log UI use this to navigate from issue to log
                 if (!string.IsNullOrEmpty(logMessage))
                 {
                     long logLineNumber = Write(WellKnownTags.Warning, logMessage);
@@ -586,9 +601,6 @@ namespace GitHub.Runner.Worker
             }
             else if (issue.Type == IssueType.Notice)
             {
-
-                // tracking line number for each issue in log file
-                // log UI use this to navigate from issue to log
                 if (!string.IsNullOrEmpty(logMessage))
                 {
                     long logLineNumber = Write(WellKnownTags.Notice, logMessage);
@@ -991,6 +1003,24 @@ namespace GitHub.Runner.Worker
             }
         }
 
+        public void WriteWebhookPayload()
+        {
+            // Makes directory for event_path data
+            var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);
+            var workflowDirectory = Path.Combine(tempDirectory, "_github_workflow");
+            Directory.CreateDirectory(workflowDirectory);
+            var gitHubEvent = GetGitHubContext("event");
+
+            // adds the GitHub event path/file if the event exists
+            if (gitHubEvent != null)
+            {
+                var workflowFile = Path.Combine(workflowDirectory, "event.json");
+                Trace.Info($"Write event payload to {workflowFile}");
+                File.WriteAllText(workflowFile, gitHubEvent, new UTF8Encoding(false));
+                SetGitHubContext("event_path", workflowFile);
+            }
+        }
+
         private void InitializeTimelineRecord(Guid timelineId, Guid timelineRecordId, Guid? parentTimelineRecordId, string recordType, string displayName, string refName, int? order)
         {
             _mainTimelineId = timelineId;
@@ -1045,6 +1075,36 @@ namespace GitHub.Runner.Worker
             var newGuid = Guid.NewGuid();
             return CreateChild(newGuid, displayName, newGuid.ToString("N"), null, null, ActionRunStage.Post, intraActionState, _childTimelineRecordOrder - Root.PostJobSteps.Count, siblingScopeName: siblingScopeName);
         }
+
+        public void ApplyContinueOnError(TemplateToken continueOnErrorToken)
+        {
+            if (Result != TaskResult.Failed)
+            {
+                return;
+            }
+            var continueOnError = false;
+            try
+            {
+                var templateEvaluator = this.ToPipelineTemplateEvaluator();
+                continueOnError = templateEvaluator.EvaluateStepContinueOnError(continueOnErrorToken, ExpressionValues, ExpressionFunctions);
+            }
+            catch (Exception ex)
+            {
+                Trace.Info("The step failed and an error occurred when attempting to determine whether to continue on error.");
+                Trace.Error(ex);
+                this.Error("The step failed and an error occurred when attempting to determine whether to continue on error.");
+                this.Error(ex);
+            }
+
+            if (continueOnError)
+            {
+                Outcome = Result;
+                Result = TaskResult.Succeeded;
+                Trace.Info($"Updated step result (continue on error)");
+            }
+
+            UpdateGlobalStepsContext();
+        }
     }
 
     // The Error/Warning/etc methods are created as extension methods to simplify unit testing.
@@ -1066,7 +1126,6 @@ namespace GitHub.Runner.Worker
             context.Error(ex.Message);
             context.Debug(ex.ToString());
         }
-
         // Do not add a format string overload. See comment on ExecutionContext.Write().
         public static void Error(this IExecutionContext context, string message)
         {

src/Runner.Worker/GitHubContext.cs

@@ -8,10 +8,10 @@ namespace GitHub.Runner.Worker
     {
         private readonly HashSet<string> _contextEnvAllowlist = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
         {
-            "action",
             "action_path",
             "action_ref",
             "action_repository",
+            "action",
             "actor",
             "api_url",
             "base_ref",
@@ -22,12 +22,12 @@ namespace GitHub.Runner.Worker
             "head_ref",
             "job",
             "path",
-            "ref",
             "ref_name",
             "ref_protected",
             "ref_type",
-            "repository",
+            "ref",
             "repository_owner",
+            "repository",
             "retention_days",
             "run_attempt",
             "run_id",
@@ -35,6 +35,7 @@ namespace GitHub.Runner.Worker
             "server_url",
             "sha",
             "step_summary",
+            "triggering_actor",
             "workflow",
             "workspace",
         };

src/Runner.Worker/Handlers/CompositeActionHandler.cs

@@ -1,8 +1,6 @@
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.Expressions2;
@@ -13,7 +11,6 @@ using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
 using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
-using GitHub.Runner.Worker;
 using GitHub.Runner.Worker.Expressions;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
@@ -407,7 +404,7 @@ namespace GitHub.Runner.Worker.Handlers
                 }
 
                 // Update context
-                SetStepsContext(step);
+                step.ExecutionContext.UpdateGlobalStepsContext();
             }
         }
 
@@ -452,6 +449,8 @@ namespace GitHub.Runner.Worker.Handlers
                 SetStepConclusion(step, Common.Util.TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value));
             }
 
+            step.ExecutionContext.ApplyContinueOnError(step.ContinueOnError);
+
             Trace.Info($"Step result: {step.ExecutionContext.Result}");
             step.ExecutionContext.Debug($"Finished: {step.DisplayName}");
             step.ExecutionContext.PublishStepTelemetry();
@@ -460,16 +459,7 @@ namespace GitHub.Runner.Worker.Handlers
         private void SetStepConclusion(IStep step, TaskResult result)
         {
             step.ExecutionContext.Result = result;
-            SetStepsContext(step);
+            step.ExecutionContext.UpdateGlobalStepsContext();
-        }
-        private void SetStepsContext(IStep step)
-        {
-            if (!string.IsNullOrEmpty(step.ExecutionContext.ContextName) && !step.ExecutionContext.ContextName.StartsWith("__", StringComparison.Ordinal))
-            {
-                // TODO: when we support continue on error, we may need to do logic here to change conclusion based on the continue on error result
-                step.ExecutionContext.Global.StepsContext.SetOutcome(step.ExecutionContext.ScopeName, step.ExecutionContext.ContextName, (step.ExecutionContext.Result ?? TaskResult.Succeeded).ToActionResult());
-                step.ExecutionContext.Global.StepsContext.SetConclusion(step.ExecutionContext.ScopeName, step.ExecutionContext.ContextName, (step.ExecutionContext.Result ?? TaskResult.Succeeded).ToActionResult());
-            }
         }
     }
 }

src/Runner.Worker/Handlers/Handler.cs

@@ -36,6 +36,7 @@ namespace GitHub.Runner.Worker.Handlers
         protected IActionCommandManager ActionCommandManager { get; private set; }
 
         public Pipelines.ActionStepDefinitionReference Action { get; set; }
+        public bool IsActionStep => Action != null; 
         public Dictionary<string, string> Environment { get; set; }
         public Variables RuntimeVariables { get; set; }
         public IExecutionContext ExecutionContext { get; set; }
@@ -49,13 +50,18 @@ namespace GitHub.Runner.Worker.Handlers
             // Print out action details
             PrintActionDetails(stage);
 
-            // Get telemetry for the action.
+            // Get telemetry for the action
-            PopulateActionTelemetry();
+            PopulateActionTelemetry(stage);
         }
 
-        protected void PopulateActionTelemetry()
+        protected void PopulateActionTelemetry(ActionRunStage stage)
         {
-            if (Action.Type == Pipelines.ActionSourceType.ContainerRegistry)
+            if (!IsActionStep)
+            {
+                ExecutionContext.StepTelemetry.Type = "runner";
+                ExecutionContext.StepTelemetry.Action = $"{stage} Job Hook";
+            }
+            else if (Action.Type == Pipelines.ActionSourceType.ContainerRegistry)
             {
                 ExecutionContext.StepTelemetry.Type = "docker";
                 var registryAction = Action as Pipelines.ContainerRegistryReference;

src/Runner.Worker/Handlers/NodeScriptActionHandler.cs

@@ -3,6 +3,8 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using GitHub.DistributedTask.Pipelines;
+using GitHub.DistributedTask.Pipelines.ContextData;
 using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
 using GitHub.Runner.Sdk;
@@ -112,6 +114,17 @@ namespace GitHub.Runner.Worker.Handlers
             // Remove environment variable that may cause conflicts with the node within the runner.
             Environment.Remove("NODE_ICU_DATA"); // https://github.com/actions/runner/issues/795
 
+            if (Data.NodeVersion == "node12" && (ExecutionContext.Global.Variables.GetBoolean(Constants.Runner.Features.Node12Warning) ?? false))
+            {
+                if (!ExecutionContext.JobContext.ContainsKey("Node12ActionsWarnings"))
+                {                     
+                    ExecutionContext.JobContext["Node12ActionsWarnings"] = new ArrayContextData();
+                }
+                var repoAction = Action as RepositoryPathReference;
+                var actionDisplayName = new StringContextData(repoAction.Name ?? repoAction.Path); // local actions don't have a 'Name'                
+                ExecutionContext.JobContext["Node12ActionsWarnings"].AssertArray("Node12ActionsWarnings").Add(actionDisplayName);
+            }
+
             using (var stdoutManager = new OutputManager(ExecutionContext, ActionCommandManager))
             using (var stderrManager = new OutputManager(ExecutionContext, ActionCommandManager))
             {

src/Runner.Worker/Handlers/ScriptHandler.cs

@@ -24,16 +24,22 @@ namespace GitHub.Runner.Worker.Handlers
 
         protected override void PrintActionDetails(ActionRunStage stage)
         {
-
+            // if we're executing a Job Extension, we won't have an 'Action'
-            if (stage == ActionRunStage.Post)
+            if (!IsActionStep)
             {
-                throw new NotSupportedException("Script action should not have 'Post' job action.");
+                if (Inputs.TryGetValue("path", out var path))
+                {
+                    ExecutionContext.Output($"##[group]Run '{path}'");
                 }
-
+                else
+                {
+                    throw new InvalidOperationException("Inputs 'path' must be set for job extensions");
+                }
+            }
+            else if (Action.Type == Pipelines.ActionSourceType.Script)
+            {
                 Inputs.TryGetValue("script", out string contents);
                 contents = contents ?? string.Empty;
-            if (Action.Type == Pipelines.ActionSourceType.Script)
-            {
                 var firstLine = contents.TrimStart(' ', '\t', '\r', '\n');
                 var firstNewLine = firstLine.IndexOfAny(new[] { '\r', '\n' });
                 if (firstNewLine >= 0)
@@ -42,18 +48,17 @@ namespace GitHub.Runner.Worker.Handlers
                 }
 
                 ExecutionContext.Output($"##[group]Run {firstLine}");
-            }
-            else
-            {
-                throw new InvalidOperationException($"Invalid action type {Action.Type} for {nameof(ScriptHandler)}");
-            }
-
                 var multiLines = contents.Replace("\r\n", "\n").TrimEnd('\n').Split('\n');
                 foreach (var line in multiLines)
                 {
                     // Bright Cyan color
                     ExecutionContext.Output($"\x1b[36;1m{line}\x1b[0m");
                 }
+            }
+            else
+            {
+                throw new InvalidOperationException($"Invalid action type {Action?.Type} for {nameof(ScriptHandler)}");
+            }
 
             string argFormat;
             string shellCommand;
@@ -132,11 +137,6 @@ namespace GitHub.Runner.Worker.Handlers
 
         public async Task RunAsync(ActionRunStage stage)
         {
-            if (stage == ActionRunStage.Post)
-            {
-                throw new NotSupportedException("Script action should not have 'Post' job action.");
-            }
-
             // Validate args
             Trace.Entering();
             ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
@@ -153,7 +153,8 @@ namespace GitHub.Runner.Worker.Handlers
             string workingDirectory = null;
             if (!Inputs.TryGetValue("workingDirectory", out workingDirectory))
             {
-                if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults))
+                // Don't use job level working directories for hooks
+                if (IsActionStep && string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults))
                 {
                     if (runDefaults.TryGetValue("working-directory", out workingDirectory))
                     {
@@ -212,7 +213,8 @@ namespace GitHub.Runner.Worker.Handlers
                 }
             }
 
-            if (!string.IsNullOrEmpty(shellCommand))
+            // Don't override runner telemetry here
+            if (!string.IsNullOrEmpty(shellCommand) && IsActionStep)
             {
                 ExecutionContext.StepTelemetry.Action = shellCommand;
             }
@@ -222,10 +224,24 @@ namespace GitHub.Runner.Worker.Handlers
             {
                 throw new ArgumentException("Invalid shell option. Shell must be a valid built-in (bash, sh, cmd, powershell, pwsh) or a format string containing '{0}'");
             }
-
+            string scriptFilePath, resolvedScriptPath;
+            if (IsActionStep)
+            {
                 // We do not not the full path until we know what shell is being used, so that we can determine the file extension
-            var scriptFilePath = Path.Combine(tempDirectory, $"{Guid.NewGuid()}{ScriptHandlerHelpers.GetScriptFileExtension(shellCommand)}");
+                scriptFilePath = Path.Combine(tempDirectory, $"{Guid.NewGuid()}{ScriptHandlerHelpers.GetScriptFileExtension(shellCommand)}");
-            var resolvedScriptPath = $"{StepHost.ResolvePathForStepHost(scriptFilePath).Replace("\"", "\\\"")}";
+                resolvedScriptPath = $"{StepHost.ResolvePathForStepHost(scriptFilePath).Replace("\"", "\\\"")}";
+            }
+            else
+            {
+                // JobExtensionRunners run a script file, we load that from the inputs here
+                if (!Inputs.ContainsKey("path"))
+                {
+                    throw new ArgumentException("Expected 'path' input to be set");
+                }
+                scriptFilePath = Inputs["path"];
+                ArgUtil.NotNullOrEmpty(scriptFilePath, "path");
+                resolvedScriptPath = Inputs["path"].Replace("\"", "\\\"");
+            }
 
             // Format arg string with script path
             var arguments = string.Format(argFormat, resolvedScriptPath);
@@ -242,8 +258,11 @@ namespace GitHub.Runner.Worker.Handlers
             // Don't add a BOM. It causes the script to fail on some operating systems (e.g. on Ubuntu 14).
             var encoding = new UTF8Encoding(false);
 #endif            
+            if (IsActionStep)
+            {
                 // Script is written to local path (ie host) but executed relative to the StepHost, which may be a container
                 File.WriteAllText(scriptFilePath, contents, encoding);
+            }
 
             // Prepend PATH
             AddPrependPathToEnvironment();

src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs

@@ -1,6 +1,8 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO;
+using GitHub.Runner.Sdk;
 
 namespace GitHub.Runner.Worker.Handlers
 {
@@ -79,5 +81,22 @@ namespace GitHub.Runner.Worker.Handlers
                 throw new ArgumentException($"Failed to parse COMMAND [..ARGS] from {shellOption}");
             }
         }
+
+        internal static string GetDefaultShellForScript(string path, Common.Tracing trace, string prependPath)
+        {
+            var format = "{0} {1}";
+            switch (Path.GetExtension(path))
+            {
+                case ".sh":
+                    // use 'sh' args but prefer bash
+                    var pathToShell = WhichUtil.Which("bash", false, trace, prependPath) ?? WhichUtil.Which("sh", true, trace, prependPath);
+                    return string.Format(format, pathToShell, _defaultArguments["sh"]);
+                case ".ps1":
+                    var pathToPowershell = WhichUtil.Which("pwsh", false, trace, prependPath) ?? WhichUtil.Which("powershell", true, trace, prependPath);
+                    return string.Format(format, pathToPowershell, _defaultArguments["powershell"]);
+                default:
+                    throw new ArgumentException($"{path} is not a valid path to a script. Make sure it ends in '.sh' or '.ps1'.");
+            }
+        }
     }
 }

src/Runner.Worker/JobExtension.cs

@@ -15,6 +15,7 @@ using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
 using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
+using GitHub.Runner.Worker;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
 namespace GitHub.Runner.Worker
@@ -248,6 +249,19 @@ namespace GitHub.Runner.Worker
                     Trace.Info("Downloading actions");
                     var actionManager = HostContext.GetService<IActionManager>();
                     var prepareResult = await actionManager.PrepareActionsAsync(context, message.Steps);
+
+                    // add hook to preJobSteps
+                    var startedHookPath = Environment.GetEnvironmentVariable("ACTIONS_RUNNER_HOOK_JOB_STARTED");
+                    if (!string.IsNullOrEmpty(startedHookPath))
+                    {
+                        var hookProvider = HostContext.GetService<IJobHookProvider>();
+                        var jobHookData = new JobHookData(ActionRunStage.Pre, startedHookPath);
+                        preJobSteps.Add(new JobExtensionRunner(runAsync: hookProvider.RunHook,
+                                                                          condition: $"{PipelineTemplateConstants.Always}()",
+                                                                          displayName: Constants.Hooks.JobStartedStepName,
+                                                                          data: (object)jobHookData));
+                    }
+
                     preJobSteps.AddRange(prepareResult.ContainerSetupSteps);
 
                     // Add start-container steps, record and stop-container steps
@@ -337,6 +351,18 @@ namespace GitHub.Runner.Worker
                         }
                     }
 
+                    // Register Job Completed hook if the variable is set
+                    var completedHookPath = Environment.GetEnvironmentVariable("ACTIONS_RUNNER_HOOK_JOB_COMPLETED");
+                    if (!string.IsNullOrEmpty(completedHookPath))
+                    {
+                        var hookProvider = HostContext.GetService<IJobHookProvider>();
+                        var jobHookData = new JobHookData(ActionRunStage.Post, completedHookPath);
+                        jobContext.RegisterPostJobStep(new JobExtensionRunner(runAsync: hookProvider.RunHook,
+                                                                          condition: $"{PipelineTemplateConstants.Always}()",
+                                                                          displayName: Constants.Hooks.JobCompletedStepName,
+                                                                          data: (object)jobHookData));
+                    }
+
                     List<IStep> steps = new List<IStep>();
                     steps.AddRange(preJobSteps);
                     steps.AddRange(jobSteps);
@@ -406,7 +432,7 @@ namespace GitHub.Runner.Worker
             // create a new timeline record node for 'Finalize job'
             IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Complete job", $"{nameof(JobExtension)}_Final", null, null, ActionRunStage.Post);
             context.StepTelemetry.Type = "runner";
-            context.StepTelemetry.Action = "complete_joh";
+            context.StepTelemetry.Action = "complete_job";
             using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
             {
                 try

src/Runner.Worker/JobHookProvider.cs

@@ -0,0 +1,95 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading.Tasks;
+using System.Linq;
+using GitHub.Runner.Common;
+using GitHub.Runner.Common.Util;
+using GitHub.Runner.Sdk;
+using GitHub.Runner.Worker.Handlers;
+
+namespace GitHub.Runner.Worker
+{
+    [ServiceLocator(Default = typeof(JobHookProvider))]
+    public interface IJobHookProvider : IRunnerService
+    {
+        Task RunHook(IExecutionContext executionContext, object data);
+    }
+
+    public class JobHookData
+    {
+        public string Path {get; private set;}
+        public ActionRunStage Stage {get; private set;}
+
+        public JobHookData(ActionRunStage stage, string path)
+        {
+            Path = path;
+            Stage = stage;
+        }
+    }
+
+    public class JobHookProvider : RunnerService, IJobHookProvider
+    {
+        public override void Initialize(IHostContext hostContext)
+        {
+            base.Initialize(hostContext);
+        }
+
+        public async Task RunHook(IExecutionContext executionContext, object data)
+        {
+            // Get Inputs
+            var hookData = data as JobHookData;
+            ArgUtil.NotNull(hookData, nameof(JobHookData));
+
+            var displayName = hookData.Stage == ActionRunStage.Pre ? "job started hook" : "job completed hook";
+            // Log to users so that they know how this step was injected
+            executionContext.Output($"A {displayName} has been configured by the self-hosted runner administrator");
+
+            // Validate script file.
+            if (!File.Exists(hookData.Path))
+            {
+                throw new FileNotFoundException("File doesn't exist");
+            }
+
+            executionContext.WriteWebhookPayload();
+
+            // Create the handler data.
+            var scriptDirectory = Path.GetDirectoryName(hookData.Path);
+            var stepHost = HostContext.CreateService<IDefaultStepHost>();
+            var prependPath = string.Join(Path.PathSeparator.ToString(), executionContext.Global.PrependPath.Reverse<string>());
+            Dictionary<string, string> inputs = new()
+            {
+                ["path"] = hookData.Path,
+                ["shell"] = ScriptHandlerHelpers.GetDefaultShellForScript(hookData.Path, Trace, prependPath)
+            };
+
+            // Create the handler
+            var handlerFactory = HostContext.GetService<IHandlerFactory>();
+            var handler = handlerFactory.Create(
+                            executionContext,
+                            action: null,
+                            stepHost,
+                            new ScriptActionExecutionData(),
+                            inputs,
+                            environment: new Dictionary<string, string>(VarUtil.EnvironmentVariableKeyComparer),
+                            executionContext.Global.Variables,
+                            actionDirectory: scriptDirectory,
+                            localActionContainerSetupSteps: null);
+            handler.PrepareExecution(hookData.Stage);
+
+            // Setup file commands
+            var fileCommandManager = HostContext.CreateService<IFileCommandManager>();
+            fileCommandManager.InitializeFiles(executionContext, null);
+
+            // Run the step and process the file commands
+            try
+            {
+                await handler.RunAsync(hookData.Stage);
+            }
+            finally
+            {
+                fileCommandManager.ProcessFiles(executionContext, executionContext.Global.Container);
+            }
+        }
+    }
+}

src/Runner.Worker/JobRunner.cs

@@ -6,6 +6,7 @@ using System.Net.Http;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using GitHub.DistributedTask.Pipelines.ContextData;
 using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
 using GitHub.Runner.Common.Util;
@@ -257,6 +258,12 @@ namespace GitHub.Runner.Worker
                 }
             }
 
+            if (jobContext.JobContext.ContainsKey("Node12ActionsWarnings"))
+            {
+                var actions = string.Join(", ", jobContext.JobContext["Node12ActionsWarnings"].AssertArray("Node12ActionsWarnings").Select(action => action.ToString()));
+                jobContext.Warning(string.Format(Constants.Runner.Node12DetectedAfterEndOfLife, actions));
+            }
+
             try
             {
                 await ShutdownQueue(throwOnFailure: true);

src/Runner.Worker/StepsRunner.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -133,8 +134,10 @@ namespace GitHub.Runner.Worker
                             // Test the condition again. The job was cancelled after the condition was originally evaluated.
                             jobCancelRegister = jobContext.CancellationToken.Register(() =>
                             {
-                                // Mark job as cancelled
+                                // Mark job as Cancelled or Failed depending on HostContext shutdown token's cancellation
-                                jobContext.Result = TaskResult.Canceled;
+                                jobContext.Result = HostContext.RunnerShutdownToken.IsCancellationRequested
+                                                    ? TaskResult.Failed
+                                                    : TaskResult.Canceled;
                                 jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
 
                                 step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'.");
@@ -172,8 +175,10 @@ namespace GitHub.Runner.Worker
                         {
                             if (jobContext.Result != TaskResult.Canceled)
                             {
-                                // Mark job as cancelled
+                                // Mark job as Cancelled or Failed depending on HostContext shutdown token's cancellation
-                                jobContext.Result = TaskResult.Canceled;
+                                jobContext.Result = HostContext.RunnerShutdownToken.IsCancellationRequested
+                                    ? TaskResult.Failed
+                                    : TaskResult.Canceled;
                                 jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
                             }
                         }
@@ -319,29 +324,8 @@ namespace GitHub.Runner.Worker
                 step.ExecutionContext.Result = TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value);
             }
 
-            // Fixup the step result if ContinueOnError
+            step.ExecutionContext.ApplyContinueOnError(step.ContinueOnError);
-            if (step.ExecutionContext.Result == TaskResult.Failed)
-            {
-                var continueOnError = false;
-                try
-                {
-                    continueOnError = templateEvaluator.EvaluateStepContinueOnError(step.ContinueOnError, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Info("The step failed and an error occurred when attempting to determine whether to continue on error.");
-                    Trace.Error(ex);
-                    step.ExecutionContext.Error("The step failed and an error occurred when attempting to determine whether to continue on error.");
-                    step.ExecutionContext.Error(ex);
-                }
 
-                if (continueOnError)
-                {
-                    step.ExecutionContext.Outcome = step.ExecutionContext.Result;
-                    step.ExecutionContext.Result = TaskResult.Succeeded;
-                    Trace.Info($"Updated step result (continue on error)");
-                }
-            }
             Trace.Info($"Step result: {step.ExecutionContext.Result}");
 
             // Complete the step context

src/Runner.Worker/action_yaml.json

@@ -129,6 +129,7 @@
                         "required": true
                     },
                     "env": "step-env",
+                    "continue-on-error": "boolean-steps-context",
                     "working-directory": "string-steps-context",
                     "shell": {
                         "type": "non-empty-string",
@@ -147,6 +148,7 @@
                         "type": "non-empty-string",
                         "required": true
                     },
+                    "continue-on-error": "boolean-steps-context",
                     "with": "step-with",
                     "env": "step-env"
                 }
@@ -201,6 +203,20 @@
             ],
             "string": {}
         },
+        "boolean-steps-context": {
+            "context": [
+                "github",
+                "inputs",
+                "strategy",
+                "matrix",
+                "steps",
+                "job",
+                "runner",
+                "env",
+                "hashFiles(1,255)"
+            ],
+            "boolean": {}
+        },
         "step-env": {
             "context": [
                 "github",

src/Sdk/Common/Common/Authentication/FederatedCredential.cs

@@ -18,6 +18,7 @@ namespace GitHub.Services.Common
 
         public override bool IsAuthenticationChallenge(IHttpResponse webResponse)
         {
+            // System.Console.WriteLine($"FederatedCredential.IsAuthenticationChallenge");
             if (webResponse == null)
             {
                 return false;

src/Sdk/Common/Common/Authentication/IssuedTokenCredential.cs

@@ -88,6 +88,7 @@ namespace GitHub.Services.Common
             IHttpResponse response,
             IssuedToken failedToken)
         {
+            // System.Console.WriteLine("IssuedTokenCredential.CreateTokenProvider");
             if (response != null && !IsAuthenticationChallenge(response))
             {
                 throw new InvalidOperationException();
@@ -99,12 +100,14 @@ namespace GitHub.Services.Common
                 {
                     throw new InvalidOperationException($"The {nameof(TokenStorageUrl)} property must have a value if the {nameof(Storage)} property is set on this instance of {GetType().Name}.");
                 }
+                // System.Console.WriteLine($"IssuedTokenCredential.CreateTokenProvider: TokenStorageUrl: {TokenStorageUrl}");
                 InitialToken = Storage.RetrieveToken(TokenStorageUrl, CredentialType);
             }
 
             IssuedTokenProvider provider = OnCreateTokenProvider(serverUrl, response);
             if (provider != null)
             {
+                // System.Console.WriteLine($"IssuedTokenCredential.CreateTokenProvider: provider: {provider}");
                 provider.TokenStorageUrl = TokenStorageUrl;
             }
 
@@ -123,6 +126,7 @@ namespace GitHub.Services.Common
 
         internal virtual string GetAuthenticationChallenge(IHttpResponse webResponse)
         {
+            // System.Console.WriteLine($"IssuedTokenCredential.GetAuthenticationChallenge");
             IEnumerable<String> values;
             if (!webResponse.Headers.TryGetValues(Internal.HttpHeaders.WwwAuthenticate, out values))
             {

src/Sdk/Common/Common/Authentication/VssCredentials.cs

@@ -108,6 +108,7 @@ namespace GitHub.Services.Common
             TaskScheduler scheduler,
             IVssCredentialPrompt credentialPrompt)
         {
+            // System.Console.WriteLine($"VssCredentials.ctor");
             this.PromptType = promptType;
 
             if (promptType == CredentialPromptType.PromptIfNeeded && scheduler == null)
@@ -150,6 +151,7 @@ namespace GitHub.Services.Common
         {
             get
             {
+                // System.Console.WriteLine($"VssCredentials.get_PromptType");
                 return m_promptType;
             }
             set
@@ -170,6 +172,7 @@ namespace GitHub.Services.Common
         {
             get
             {
+                // System.Console.WriteLine($"VssCredentials.get_Federated");
                 return m_federatedCredential;
             }
         }
@@ -184,6 +187,7 @@ namespace GitHub.Services.Common
         {
             get
             {
+                // System.Console.WriteLine($"VssCredentials.get_Storage");
                 return m_credentialStorage;
             }
             set
@@ -203,6 +207,7 @@ namespace GitHub.Services.Common
         /// </summary>
         internal virtual bool TryGetValidAdalToken(IVssCredentialPrompt prompt)
         {
+            // System.Console.WriteLine($"VssCredentials.TryGetValidAdalToken");
             return false;
         }
 
@@ -218,6 +223,7 @@ namespace GitHub.Services.Common
             IHttpResponse webResponse,
             IssuedToken failedToken)
         {
+            // System.Console.WriteLine("VssCredential.CreateTokenProvider");
             ArgumentUtility.CheckForNull(serverUrl, "serverUrl");
 
             IssuedTokenProvider tokenProvider = null;
@@ -263,6 +269,7 @@ namespace GitHub.Services.Common
             Uri serverUrl,
             out IssuedTokenProvider provider)
         {
+            // System.Console.WriteLine($"VssCredentials.TryGetTokenProvider");
             ArgumentUtility.CheckForNull(serverUrl, "serverUrl");
 
             lock (m_thisLock)
@@ -272,11 +279,13 @@ namespace GitHub.Services.Common
                 {
                     if (m_federatedCredential != null)
                     {
+                        // System.Console.WriteLine($"VssCredentials.TryGetTokenProvider: Using federated credential");
                         m_currentProvider = m_federatedCredential.CreateTokenProvider(serverUrl, null, null);
                     }
 
                     if (m_currentProvider != null)
                     {
+                        // System.Console.WriteLine($"VssCredentials.TryGetTokenProvider: Issued token provider created");
                         VssHttpEventSource.Log.IssuedTokenProviderCreated(VssTraceActivity.Current, m_currentProvider);
                     }
                 }
@@ -294,6 +303,7 @@ namespace GitHub.Services.Common
         /// <returns>True if this is an token authentication redirect, false otherwise</returns>
         internal bool IsAuthenticationChallenge(IHttpResponse webResponse)
         {
+            // System.Console.WriteLine($"VssCredentials.IsAuthenticationChallenge");
             if (webResponse == null)
             {
                 return false;
@@ -313,6 +323,7 @@ namespace GitHub.Services.Common
             Uri serviceLocation,
             string identityProvider)
         {
+            // System.Console.WriteLine($"VssCredentials.SignOut");
             // Remove the token in the storage and the current token provider. Note that we don't
             // call InvalidateToken here because we want to remove the whole token not just its value
             if ((m_currentProvider != null) && (m_currentProvider.CurrentToken != null))
@@ -349,6 +360,7 @@ namespace GitHub.Services.Common
             string token,
             IDictionary<string, string> attributes)
         {
+            // System.Console.WriteLine($"VssCredentials.WriteAuthorizationToken");
             int i = 0;
             for (int j = 0; j < token.Length; i++, j += 128)
             {
@@ -360,6 +372,7 @@ namespace GitHub.Services.Common
 
         protected static string ReadAuthorizationToken(IDictionary<string, string> attributes)
         {
+            // System.Console.WriteLine($"VssCredentials.ReadAuthorizationToken");
             string authTokenCountValue;
             if (attributes.TryGetValue("AuthTokenSegmentCount", out authTokenCountValue))
             {

src/Sdk/Common/Common/VssHttpMessageHandler.cs

@@ -49,6 +49,7 @@ namespace GitHub.Services.Common
             VssHttpRequestSettings settings,
             HttpMessageHandler innerHandler)
         {
+            // System.Console.WriteLine("VssHttpMessageHandler.ctor");
             this.Credentials = credentials;
             this.Settings = settings;
             this.ExpectContinue = settings.ExpectContinue;
@@ -122,6 +123,7 @@ namespace GitHub.Services.Common
             HttpRequestMessage request,
             CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("VssHttpMessageHandler.SendAsync");
             VssTraceActivity traceActivity = VssTraceActivity.Current;
 
             var traceInfo = VssHttpMessageHandlerTraceInfo.GetTraceInfo(request);
@@ -130,6 +132,7 @@ namespace GitHub.Services.Common
             if (!m_appliedClientCertificatesToTransportHandler &&
                 request.RequestUri.Scheme == "https")
             {
+                // System.Console.WriteLine("VssHttpMessageHandler.SendAsync: !appliedClientCertificatesToTransportHandler");
                 HttpClientHandler httpClientHandler = m_transportHandler as HttpClientHandler;
                 if (httpClientHandler != null &&
                     this.Settings.ClientCertificateManager != null &&
@@ -144,6 +147,7 @@ namespace GitHub.Services.Common
             if (!m_appliedServerCertificateValidationCallbackToTransportHandler &&
                 request.RequestUri.Scheme == "https")
             {
+                // System.Console.WriteLine("VssHttpMessageHandler.SendAsync: !appliedServerCertificateValidationCallbackToTransportHandler");
                 HttpClientHandler httpClientHandler = m_transportHandler as HttpClientHandler;
                 if (httpClientHandler != null &&
                     this.Settings.ServerCertificateValidationCallback != null)
@@ -165,6 +169,7 @@ namespace GitHub.Services.Common
             IssuedTokenProvider provider;
             if (this.Credentials.TryGetTokenProvider(request.RequestUri, out provider))
             {
+                // System.Console.WriteLine("VssHttpMessageHandler.SendAsync: Got token provider from credentials");
                 token = provider.CurrentToken;
             }
 
@@ -225,6 +230,7 @@ namespace GitHub.Services.Common
 
                     traceInfo?.TraceResponseContentTime();
 
+                    // System.Console.WriteLine($"VssHttpMessageHandler.SendAsync: Creating response wrapper");
                     responseWrapper = new HttpResponseMessageWrapper(response);
 
                     if (!this.Credentials.IsAuthenticationChallenge(responseWrapper))
@@ -232,6 +238,7 @@ namespace GitHub.Services.Common
                         // Validate the token after it has been successfully authenticated with the server.
                         if (provider != null)
                         {
+                            // System.Console.WriteLine("VssHttpMessageHandler.SendAsync: Validating token");
                             provider.ValidateToken(token, responseWrapper);
                         }
 
@@ -243,6 +250,7 @@ namespace GitHub.Services.Common
                     }
                     else
                     {
+                        // System.Console.WriteLine($"VssHttpMessageHandler.SendAsync: Auth challenge. Response status code {response.StatusCode}; headers {response.Headers}");
                         // In the case of a Windows token, only apply it to the web proxy if it
                         // returned a 407 Proxy Authentication Required. If we didn't get this
                         // status code back, then the proxy (if there is one) is clearly working fine,
@@ -288,6 +296,7 @@ namespace GitHub.Services.Common
                         }
 
                         // Now invoke the provider and await the result
+                        // System.Console.WriteLine($"VssHttpMessageHandler.SendAsync: Calling GetTokenAsync");
                         token = await provider.GetTokenAsync(token, tokenSource.Token).ConfigureAwait(false);
 
                         // I always see 0 here, but the method above could take more time so keep for now
@@ -432,6 +441,7 @@ namespace GitHub.Services.Common
                     activity != VssTraceActivity.Empty &&
                     !request.Headers.Contains(HttpHeaders.TfsSessionHeader))
                 {
+                    // System.Console.WriteLine($"VssHttpMessageHandler.ApplyHeaders: Activity ID {activity.Id}");
                     request.Headers.Add(HttpHeaders.TfsSessionHeader, activity.Id.ToString("D"));
                 }
 
@@ -452,13 +462,16 @@ namespace GitHub.Services.Common
             ICredentials credentialsToken = token as ICredentials;
             if (credentialsToken != null)
             {
+                // System.Console.WriteLine("VssHttpMessageHandler.ApplyToken: Credentials token != null");
                 if (applyICredentialsToWebProxy)
                 {
+                    // System.Console.WriteLine("VssHttpMessageHandler.ApplyToken: Apply credentials to web proxy");
                     HttpClientHandler httpClientHandler = m_transportHandler as HttpClientHandler;
 
                     if (httpClientHandler != null &&
                         httpClientHandler.Proxy != null)
                     {
+                        // System.Console.WriteLine("VssHttpMessageHandler.ApplyToken: Setting proxy crednetials");
                         httpClientHandler.Proxy.Credentials = credentialsToken;
                     }
                 }
@@ -467,6 +480,7 @@ namespace GitHub.Services.Common
             }
             else
             {
+                // System.Console.WriteLine("VssHttpMessageHandler.ApplyToken: Applying credentials to request");
                 token.ApplyTo(new HttpRequestMessageWrapper(request));
             }
         }
@@ -479,7 +493,8 @@ namespace GitHub.Services.Common
             HttpClientHandler httpClientHandler = handler as HttpClientHandler;
             if (httpClientHandler != null)
             {
-                httpClientHandler.AllowAutoRedirect = settings.AllowAutoRedirect;
+                // System.Console.WriteLine($"VssHttpMessageHandler.ApplySettings: Default credentials = {defaultCredentials} AllowAutoRedirect = {settings.AllowAutoRedirect}");
+                httpClientHandler.AllowAutoRedirect = true; //settings.AllowAutoRedirect;
                 httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
                 //Setting httpClientHandler.UseDefaultCredentials to false in .Net Core, clears httpClientHandler.Credentials if
                 //credentials is already set to defaultcredentials. Therefore httpClientHandler.Credentials must be 
@@ -550,6 +565,7 @@ namespace GitHub.Services.Common
                 Uri uri,
                 String authType)
             {
+                // System.Console.WriteLine($"CredentialWrapper.GetCredential: InnerCredentials = {InnerCredentials}");
                 return InnerCredentials != null ? InnerCredentials.GetCredential(uri, authType) : null;
             }
         }

src/Sdk/DTGenerated/Generated/TaskAgentHttpClientBase.cs

@@ -27,6 +27,7 @@ using System.Net.Http.Headers;
 using System.Net.Http.Formatting;
 using System.Threading;
 using System.Threading.Tasks;
+using GitHub.DistributedTask.Pipelines;
 using GitHub.Services.Common;
 using GitHub.Services.WebApi;
 
@@ -378,6 +379,7 @@ namespace GitHub.DistributedTask.WebApi
         /// <param name="requestId"></param>
         /// <param name="lockToken"></param>
         /// <param name="request"></param>
+        /// <param name="targetHostId"></param>
         /// <param name="userState"></param>
         /// <param name="cancellationToken">The cancellation token to cancel operation.</param>
         [EditorBrowsable(EditorBrowsableState.Never)]
@@ -386,6 +388,7 @@ namespace GitHub.DistributedTask.WebApi
             long requestId,
             Guid lockToken,
             TaskAgentJobRequest request,
+            Guid targetHostId,
             object userState = null,
             CancellationToken cancellationToken = default)
         {
@@ -396,6 +399,7 @@ namespace GitHub.DistributedTask.WebApi
 
             List<KeyValuePair<string, string>> queryParams = new List<KeyValuePair<string, string>>();
             queryParams.Add("lockToken", lockToken.ToString());
+            queryParams.Add("targetHostId", targetHostId.ToString());
 
             return SendAsync<TaskAgentJobRequest>(
                 httpMethod,
@@ -703,6 +707,47 @@ namespace GitHub.DistributedTask.WebApi
                 cancellationToken: cancellationToken);
         }
 
+        /// <summary>
+        /// [Preview API]
+        /// </summary>
+        /// <param name="scopeId"></param>
+        /// <param name="planType"></param>
+        /// <param name="planGroup"></param>
+        /// <param name="planId"></param>
+        /// <param name="instanceRefsJson"></param>
+        /// <param name="userState"></param>
+        /// <param name="cancellationToken">The cancellation token to cancel operation.</param>
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        public virtual Task<Pipelines.AgentJobRequestMessage> GetJobMessageAsync(
+            Guid scopeId,
+            Guid hostId,
+            string planType,
+            string planGroup,
+            Guid planId,
+            string instanceRefsJson,
+            object userState = null,
+            CancellationToken cancellationToken = default)
+        {
+            HttpMethod httpMethod = new HttpMethod("GET");
+            Guid locationId = new Guid("25adab70-1379-4186-be8e-b643061ebe3a");
+
+            List<KeyValuePair<string, string>> queryParams = new List<KeyValuePair<string, string>>();
+            queryParams.Add("scopeId", scopeId.ToString());
+            queryParams.Add("hostId", hostId.ToString());
+            queryParams.Add("planType", planType);
+            queryParams.Add("planGroup", planGroup);
+            queryParams.Add("planId", planId.ToString());
+            queryParams.Add("instanceRefsJson", instanceRefsJson);
+
+            return SendAsync<Pipelines.AgentJobRequestMessage>(
+                httpMethod,
+                locationId,
+                version: new ApiResourceVersion(6.0, 1),
+                queryParameters: queryParams,
+                userState: userState,
+                cancellationToken: cancellationToken);
+        }
+
         /// <summary>
         /// [Preview API]
         /// </summary>
@@ -717,6 +762,7 @@ namespace GitHub.DistributedTask.WebApi
             object userState = null,
             CancellationToken cancellationToken = default)
         {
+            // System.Console.WriteLine("TaskAgentHttpClientBase.CreateAgentSessionAsync");
             HttpMethod httpMethod = new HttpMethod("POST");
             Guid locationId = new Guid("134e239e-2df3-4794-a6f6-24f1f19ec8dc");
             object routeValues = new { poolId = poolId };

src/Sdk/DTWebApi/WebApi/JobRequestMessageTypes.cs

@@ -5,5 +5,6 @@ namespace GitHub.DistributedTask.WebApi
     public static class JobRequestMessageTypes
     {
         public const String PipelineAgentJobRequest = "PipelineAgentJobRequest";
+        public const String RunnerJobRequest = "RunnerJobRequest";
     }
 }

src/Sdk/DTWebApi/WebApi/TaskAgentHttpClient.cs

@@ -64,6 +64,7 @@ namespace GitHub.DistributedTask.WebApi
             Guid lockToken,
             DateTime finishTime,
             TaskResult result,
+            Guid targetHostId,
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
@@ -74,7 +75,7 @@ namespace GitHub.DistributedTask.WebApi
                 Result = result,
             };
 
-            return UpdateAgentRequestAsync(poolId, requestId, lockToken, request, userState, cancellationToken);
+            return UpdateAgentRequestAsync(poolId, requestId, lockToken, request, targetHostId, userState, cancellationToken);
         }
 
         public Task<List<TaskAgent>> GetAgentsAsync(
@@ -152,6 +153,7 @@ namespace GitHub.DistributedTask.WebApi
             CancellationToken cancellationToken = default(CancellationToken),
             Func<HttpResponseMessage, CancellationToken, Task<T>> processResponse = null)
         {
+            // System.Console.WriteLine("TaskAgentHttpClient.SendAsync 1");
             return SendAsync<T>(method, null, locationId, routeValues, version, content, queryParameters, userState, cancellationToken, processResponse);
         }
 
@@ -170,6 +172,7 @@ namespace GitHub.DistributedTask.WebApi
             using (VssTraceActivity.GetOrCreate().EnterCorrelationScope())
             using (HttpRequestMessage requestMessage = await CreateRequestMessageAsync(method, additionalHeaders, locationId, routeValues, version, content, queryParameters, userState, cancellationToken).ConfigureAwait(false))
             {
+                // System.Console.WriteLine("TaskAgentHttpClient.SendAsync 2");
                 return await SendAsync<T>(requestMessage, userState, cancellationToken, processResponse).ConfigureAwait(false);
             }
         }
@@ -180,6 +183,7 @@ namespace GitHub.DistributedTask.WebApi
             CancellationToken cancellationToken = default(CancellationToken),
             Func<HttpResponseMessage, CancellationToken, Task<T>> processResponse = null)
         {
+            // System.Console.WriteLine("TaskAgentHttpClient.SendAsync 3");
             if (processResponse == null)
             {
                 processResponse = ReadContentAsAsync<T>;

src/Sdk/WebApi/WebApi/Location/ServerDataProvider.cs

@@ -801,6 +801,7 @@ namespace GitHub.Services.WebApi.Location
 
         private async Task<ConnectionData> GetConnectionDataAsync(ConnectOptions connectOptions, int lastChangeId, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("ServerDataProvider.GetConnectionDataAsync");
             int timeoutRetries = 1;
 
             while (true)

src/Sdk/WebApi/WebApi/OAuth/VssOAuthAccessToken.cs

@@ -75,6 +75,7 @@ namespace GitHub.Services.OAuth
 
         internal override void ApplyTo(IHttpRequest request)
         {
+            // System.Console.WriteLine($"VssOAuthAccessToken.ApplyTo: Bearer {m_value}");
             request.Headers.SetValue(Common.Internal.HttpHeaders.Authorization, $"Bearer {m_value}");
         }
 

src/Sdk/WebApi/WebApi/OAuth/VssOAuthAccessTokenCredential.cs

@@ -60,6 +60,7 @@ namespace GitHub.Services.OAuth
             Uri serverUrl,
             IHttpResponse response)
         {
+            // System.Console.WriteLine($"VssOAuthAccessTokenCredential.OnCreateTokenProvider");
             return new VssOAuthAccessTokenProvider(this, serverUrl, null);
         }
 
@@ -71,6 +72,7 @@ namespace GitHub.Services.OAuth
                 Uri signInUrl) 
                 : base(credential, serverUrl, signInUrl)
             { 
+                // System.Console.WriteLine($"VssOAuthAccessTokenProvider.ctor");
             }
 
             public override Boolean GetTokenIsInteractive

src/Sdk/WebApi/WebApi/OAuth/VssOAuthCredential.cs

@@ -103,17 +103,23 @@ namespace GitHub.Services.OAuth
         /// <returns>True if the web response indicates an authorization challenge; otherwise, false</returns>
         public override Boolean IsAuthenticationChallenge(IHttpResponse webResponse)
         {
+            // System.Console.WriteLine($"VssOAuthCredential.IsAuthenticationChallenge");
             if (webResponse == null)
             {
+                // System.Console.WriteLine($"VssOAuthCredential.IsAuthenticationChallenge: webResponse is null");
                 return false;
             }
 
             if (webResponse.StatusCode == HttpStatusCode.Found ||
                 webResponse.StatusCode == HttpStatusCode.Unauthorized)
             {
-                return webResponse.Headers.GetValues(Common.Internal.HttpHeaders.WwwAuthenticate).Any(x => x.IndexOf("Bearer", StringComparison.OrdinalIgnoreCase) >= 0);
+                // System.Console.WriteLine($"VssOAuthCredential.IsAuthenticationChallenge: found or unauthorized");
+                var result = webResponse.Headers.GetValues(Common.Internal.HttpHeaders.WwwAuthenticate).Any(x => x.IndexOf("Bearer", StringComparison.OrdinalIgnoreCase) >= 0);
+                // System.Console.WriteLine($"VssOAuthCredential.IsAuthenticationChallenge: {result}");
+                return result;
             }
 
+            // System.Console.WriteLine($"VssOAuthCredential.IsAuthenticationChallenge: false");
             return false;
         }
 
@@ -121,6 +127,7 @@ namespace GitHub.Services.OAuth
             Uri serverUrl, 
             IHttpResponse response)
         {
+            // System.Console.WriteLine($"VssOAuthCredential.OnCreateTokenProvider");
             return new VssOAuthTokenProvider(this, serverUrl);
         }
 

src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenHttpClient.cs

@@ -54,6 +54,7 @@ namespace GitHub.Services.OAuth
             VssOAuthTokenParameters tokenParameters = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // todo: qqq
             VssTraceActivity traceActivity = VssTraceActivity.Current;
             using (HttpClient client = new HttpClient(CreateMessageHandler(this.AuthorizationUrl)))
             {

src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenProvider.cs

@@ -47,6 +47,7 @@ namespace GitHub.Services.OAuth
             VssOAuthTokenParameters tokenParameters)
             : base(credential, serverUrl, authorizationUrl)
         {
+            // System.Console.WriteLine($"VssOAuthTokenProvider.ctor");
             m_grant = grant;
             m_tokenParameters = tokenParameters;
             m_clientCredential = clientCrential;
@@ -59,6 +60,7 @@ namespace GitHub.Services.OAuth
         {
             get
             {
+                // System.Console.WriteLine($"VssOAuthTokenProvider.get_Grant");
                 return m_grant;
             }
         }
@@ -70,6 +72,7 @@ namespace GitHub.Services.OAuth
         {
             get
             {
+                // System.Console.WriteLine($"VssOAuthTokenProvider.get_ClientCredential");
                 return m_clientCredential;
             }
         }
@@ -81,6 +84,7 @@ namespace GitHub.Services.OAuth
         {
             get
             {
+                // System.Console.WriteLine($"VssOAuthTokenProvider.get_TokenParameters");
                 return m_tokenParameters;
             }
         }
@@ -92,6 +96,7 @@ namespace GitHub.Services.OAuth
         {
             get
             {
+                // System.Console.WriteLine($"VssOAuthTokenProvider.get_GetTokenIsInteractive");
                 return false;
             }
         }
@@ -100,6 +105,7 @@ namespace GitHub.Services.OAuth
         {
             get
             {
+                // System.Console.WriteLine($"VssOAuthTokenProvider.get_AuthenticationParameter");
                 if (this.ClientCredential == null)
                 {
                     return null;
@@ -115,12 +121,14 @@ namespace GitHub.Services.OAuth
         {
             get
             {
+                // System.Console.WriteLine($"VssOAuthTokenProvider.get_AuthenticationScheme");
                 return "Bearer";
             }
         }
 
         public async Task<string> ValidateCredentialAsync(CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine($"VssOAuthTokenProvider.ValidateCredentialAsync: Calling VssOAuthTokenHttpClient.GetTokenAsync");
             var tokenHttpClient = new VssOAuthTokenHttpClient(this.SignInUrl);
             var tokenResponse = await tokenHttpClient.GetTokenAsync(this.Grant, this.ClientCredential, this.TokenParameters, cancellationToken);
 
@@ -139,6 +147,7 @@ namespace GitHub.Services.OAuth
             IssuedToken failedToken,
             CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine($"VssOAuthTokenProvider.OnGetTokenAsync");
             if (this.SignInUrl == null ||
                 this.Grant == null ||
                 this.ClientCredential == null)
@@ -151,6 +160,7 @@ namespace GitHub.Services.OAuth
             try
             {
                 var tokenHttpClient = new VssOAuthTokenHttpClient(this.SignInUrl);
+                // System.Console.WriteLine($"VssOAuthTokenProvider.OnGetTokenAsync: Calling VssOAuthTokenHttpClient.GetTokenAsync; sign-in url {this.SignInUrl.AbsoluteUri}");
                 var tokenResponse = await tokenHttpClient.GetTokenAsync(this.Grant, this.ClientCredential, this.TokenParameters, cancellationToken).ConfigureAwait(false);
                 if (!String.IsNullOrEmpty(tokenResponse.AccessToken))
                 {
@@ -197,6 +207,7 @@ namespace GitHub.Services.OAuth
 
         protected virtual IssuedToken CreateIssuedToken(VssOAuthTokenResponse tokenResponse)
         {
+            // System.Console.WriteLine($"VssOAuthTokenProvider.CreateIssuedToken");
             if (tokenResponse.ExpiresIn > 0)
             {
                 return new VssOAuthAccessToken(tokenResponse.AccessToken, DateTime.UtcNow.AddSeconds(tokenResponse.ExpiresIn));

src/Sdk/WebApi/WebApi/VssConnection.cs

@@ -100,6 +100,7 @@ namespace GitHub.Services.WebApi
             IDictionary<String, String> parameters,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssConnection.ConnectAsync");
             CheckForDisposed();
             // Set the connectMode on the credential's FederatedPrompt
             if (Credentials.Federated != null && Credentials.Federated.Prompt != null)

src/Sdk/WebApi/WebApi/VssHttpClientBase.cs

@@ -390,6 +390,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 1");
             return SendAsync<T>(method, null, locationId, routeValues, version, content, queryParameters, userState, cancellationToken);
         }
 
@@ -404,6 +405,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 2");
             using (VssTraceActivity.GetOrCreate().EnterCorrelationScope())
             using (HttpRequestMessage requestMessage = await CreateRequestMessageAsync(method, additionalHeaders, locationId, routeValues, version, content, queryParameters, userState, cancellationToken).ConfigureAwait(false))
             {
@@ -422,6 +424,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 3");
             using (VssTraceActivity.GetOrCreate().EnterCorrelationScope())
             using (HttpRequestMessage requestMessage = await CreateRequestMessageAsync(method, additionalHeaders, locationId, routeValues, version, content, queryParameters, userState, cancellationToken).ConfigureAwait(false))
             {
@@ -455,6 +458,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 4");
             using (VssTraceActivity.GetOrCreate().EnterCorrelationScope())
             using (HttpRequestMessage requestMessage = await CreateRequestMessageAsync(method, locationId, routeValues, version, content, queryParameters, userState, cancellationToken).ConfigureAwait(false))
             {
@@ -473,6 +477,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 5");
             using (VssTraceActivity.GetOrCreate().EnterCorrelationScope())
             using (HttpRequestMessage requestMessage = await CreateRequestMessageAsync(method, locationId, routeValues, version, content, queryParameters, userState, cancellationToken).ConfigureAwait(false))
             {
@@ -501,6 +506,7 @@ namespace GitHub.Services.WebApi
             CancellationToken cancellationToken = default(CancellationToken),
             String mediaType = c_jsonMediaType)
         {
+            // System.Console.WriteLine("VssHttpClientBase.CreateRequestMessageAsync 1");
             return CreateRequestMessageAsync(method, null, locationId, routeValues, version, content, queryParameters, userState, cancellationToken, mediaType);
         }
 
@@ -526,6 +532,7 @@ namespace GitHub.Services.WebApi
             CancellationToken cancellationToken = default(CancellationToken),
             String mediaType = c_jsonMediaType)
         {
+            // System.Console.WriteLine("VssHttpClientBase.CreateRequestMessageAsync 2");
             // Lookup the location
             ApiResourceLocation location = await GetResourceLocationAsync(locationId, userState, cancellationToken).ConfigureAwait(false);
             if (location == null)
@@ -555,6 +562,7 @@ namespace GitHub.Services.WebApi
             IEnumerable<KeyValuePair<String, String>> queryParameters = null,
             String mediaType = c_jsonMediaType)
         {
+            // System.Console.WriteLine("VssHttpClientBase.CreateRequestMessageAsync 3");
             return CreateRequestMessage(method, null, location, routeValues, version, content, queryParameters, mediaType);
         }
 
@@ -578,6 +586,7 @@ namespace GitHub.Services.WebApi
             IEnumerable<KeyValuePair<String, String>> queryParameters = null,
             String mediaType = c_jsonMediaType)
         {
+            // System.Console.WriteLine("VssHttpClientBase.CreateRequestMessageAsync 4");
             CheckForDisposed();
             // Negotiate the request version to send
             ApiResourceVersion requestVersion = NegotiateRequestVersion(location, version);
@@ -749,12 +758,14 @@ namespace GitHub.Services.WebApi
             //from deadlocking...
             using (HttpResponseMessage response = await this.SendAsync(message, userState, cancellationToken).ConfigureAwait(false))
             {
+                // System.Console.WriteLine("VssHttpClientBase.SendAsync 6");
                 return await ReadContentAsAsync<T>(response, cancellationToken).ConfigureAwait(false);
             }
         }
 
         protected async Task<T> ReadContentAsAsync<T>(HttpResponseMessage response, CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine($"VssHttpClientBase.ReadContentAsAsync {response.Headers}");
             CheckForDisposed();
             Boolean isJson = IsJsonResponse(response);
             bool mismatchContentType = false;
@@ -766,17 +777,20 @@ namespace GitHub.Services.WebApi
                     !typeof(Byte[]).GetTypeInfo().IsAssignableFrom(typeof(T).GetTypeInfo()) &&
                     !typeof(JObject).GetTypeInfo().IsAssignableFrom(typeof(T).GetTypeInfo()))
                 {
+                    // System.Console.WriteLine("VssHttpClientBase.ReadContentAsAsync: isJson 1");
                     // expect it to come back wrapped, if it isn't it is a bug!
                     var wrapper = await ReadJsonContentAsync<VssJsonCollectionWrapper<T>>(response, cancellationToken).ConfigureAwait(false);
                     return wrapper.Value;
                 }
                 else if (isJson)
                 {
+                    // System.Console.WriteLine("VssHttpClientBase.ReadContentAsAsync: isJson 2");
                     return await ReadJsonContentAsync<T>(response, cancellationToken).ConfigureAwait(false);
                 }
             }
             catch (JsonReaderException)
             {
+                // System.Console.WriteLine("VssHttpClientBase.ReadContentAsAsync: mismatchContentType");
                 // We thought the content was JSON but failed to parse. 
                 // In this case, do nothing and utilize the HandleUnknownContentType call below
                 mismatchContentType = true;
@@ -802,6 +816,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 7");
             // the default in httpClient for HttpCompletionOption is ResponseContentRead so that is what we do here
             return this.SendAsync(
                 message,
@@ -816,6 +831,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 8");
             CheckForDisposed();
             if (message.Headers.UserAgent != null)
             {
@@ -851,6 +867,7 @@ namespace GitHub.Services.WebApi
                 //ConfigureAwait(false) enables the continuation to be run outside
                 //any captured SyncronizationContext (such as ASP.NET's) which keeps things
                 //from deadlocking...
+                // System.Console.WriteLine($"VssHttpClientBase.SendAsync 8: Calling Client.SendAsync {message}");
                 HttpResponseMessage response = await Client.SendAsync(message, completionOption, cancellationToken).ConfigureAwait(false);
 
                 // Inject delay or failure for testing
@@ -868,6 +885,7 @@ namespace GitHub.Services.WebApi
         [Obsolete("Use VssHttpClientBase.HandleResponseAsync instead")]
         protected virtual void HandleResponse(HttpResponseMessage response)
         {
+            // System.Console.WriteLine("VssHttpClientBase.HandleResponse 1");
 
         }
 
@@ -875,6 +893,7 @@ namespace GitHub.Services.WebApi
             HttpResponseMessage response,
             CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine($"VssHttpClientBase.HandleResponse 2 status code {response.StatusCode} headers {response.Headers}");
             response.Trace();
             VssHttpEventSource.Log.HttpRequestStop(VssTraceActivity.Current, response);
 
@@ -886,6 +905,7 @@ namespace GitHub.Services.WebApi
             }
             else if (ShouldThrowError(response))
             {
+                // System.Console.WriteLine("VssHttpClientBase.HandleResponse: Should throw error");
                 Exception exToThrow = null;
                 if (IsJsonResponse(response))
                 {
@@ -909,6 +929,7 @@ namespace GitHub.Services.WebApi
                     {
                         message = response.ReasonPhrase;
                     }
+                    // System.Console.WriteLine($"VssHttpClientBase.HandleResponse: Exception message {message}");
                     exToThrow = new VssServiceResponseException(response.StatusCode, message, exToThrow);
                 }
 

src/Test/L0/Worker/ActionRunnerL0.cs

@@ -118,7 +118,7 @@ namespace GitHub.Runner.Common.Tests.Worker
             await _actionRunner.RunAsync();
 
             //Assert
-            _ec.Verify(x => x.SetGitHubContext("event_path", Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "_github_workflow", "event.json")), Times.Once);
+            _ec.Verify(x => x.WriteWebhookPayload(), Times.Once);
         }
 
         [Fact]

src/Test/L0/Worker/ExecutionContextL0.cs

@@ -8,6 +8,7 @@ using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Worker;
 using Moq;
 using Xunit;
+using GitHub.DistributedTask.ObjectTemplating.Tokens;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
 namespace GitHub.Runner.Common.Tests.Worker
@@ -90,6 +91,63 @@ namespace GitHub.Runner.Common.Tests.Worker
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public void ApplyContinueOnError_CheckResultAndOutcome()
+        {
+            using (TestHostContext hc = CreateTestContext())
+            {
+
+                // Arrange: Create a job request message.
+                TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
+                TimelineReference timeline = new TimelineReference();
+                Guid jobId = Guid.NewGuid();
+                string jobName = "some job name";
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
+                {
+                    Alias = Pipelines.PipelineConstants.SelfAlias,
+                    Id = "github",
+                    Version = "sha1"
+                });
+                jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();
+                jobRequest.Variables["ACTIONS_STEP_DEBUG"] = "true";
+
+                // Arrange: Setup the paging logger.
+                var pagingLogger = new Mock<IPagingLogger>();
+                var jobServerQueue = new Mock<IJobServerQueue>();
+                jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny<Guid>(), It.IsAny<TimelineRecord>()));
+                jobServerQueue.Setup(x => x.QueueWebConsoleLine(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<long>())).Callback((Guid id, string msg, long? lineNumber) => { hc.GetTrace().Info(msg); });
+
+                hc.EnqueueInstance(pagingLogger.Object);
+                hc.SetSingleton(jobServerQueue.Object);
+
+                var ec = new Runner.Worker.ExecutionContext();
+                ec.Initialize(hc);
+
+                // Act.
+                ec.InitializeJob(jobRequest, CancellationToken.None);
+
+                foreach (var tc in new List<(TemplateToken token, TaskResult result, TaskResult? expectedResult, TaskResult? expectedOutcome)> {
+                (token: new BooleanToken(null, null, null, true), result: TaskResult.Failed, expectedResult: TaskResult.Succeeded, expectedOutcome: TaskResult.Failed),
+                (token: new BooleanToken(null, null, null, true), result: TaskResult.Succeeded, expectedResult: TaskResult.Succeeded, expectedOutcome: null),
+                (token: new BooleanToken(null, null, null, true), result: TaskResult.Canceled, expectedResult: TaskResult.Canceled, expectedOutcome: null),
+                (token: new BooleanToken(null, null, null, false), result: TaskResult.Failed, expectedResult: TaskResult.Failed, expectedOutcome: null),
+                (token: new BooleanToken(null, null, null, false), result: TaskResult.Succeeded, expectedResult: TaskResult.Succeeded, expectedOutcome: null),
+                (token: new BooleanToken(null, null, null, false), result: TaskResult.Canceled, expectedResult: TaskResult.Canceled, expectedOutcome: null),
+            })
+                {
+                    ec.Result = tc.result;
+                    ec.Outcome = null;
+                    ec.ApplyContinueOnError(tc.token);
+                    Assert.Equal(ec.Result, tc.expectedResult);
+                    Assert.Equal(ec.Outcome, tc.expectedOutcome);
+                }
+            }
+        }
+
+
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]
@@ -144,6 +202,55 @@ namespace GitHub.Runner.Common.Tests.Worker
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public void AddIssue_AddStepAndLineNumberInformation()
+        {
+            using (TestHostContext hc = CreateTestContext())
+            {
+
+                TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
+                TimelineReference timeline = new TimelineReference();
+                Guid jobId = Guid.NewGuid();
+                string jobName = "some job name";
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
+                {
+                    Alias = Pipelines.PipelineConstants.SelfAlias,
+                    Id = "github",
+                    Version = "sha1"
+                });
+                jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();
+
+                // Arrange: Setup the paging logger.
+                var pagingLogger = new Mock<IPagingLogger>();
+                var pagingLogger2 = new Mock<IPagingLogger>();
+                var jobServerQueue = new Mock<IJobServerQueue>();
+                jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny<Guid>(), It.IsAny<TimelineRecord>()));
+
+                hc.EnqueueInstance(pagingLogger.Object);
+                hc.EnqueueInstance(pagingLogger2.Object);
+                hc.SetSingleton(jobServerQueue.Object);
+
+                var ec = new Runner.Worker.ExecutionContext();
+                ec.Initialize(hc);
+                ec.InitializeJob(jobRequest, CancellationToken.None);
+                ec.Start();
+
+                var embeddedStep = ec.CreateChild(Guid.NewGuid(), "action_1_pre", "action_1_pre", null, null, ActionRunStage.Main, isEmbedded: true);
+                embeddedStep.Start();
+
+                embeddedStep.AddIssue(new Issue() { Type = IssueType.Error, Message = "error annotation that should have step and line number information" });
+                embeddedStep.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning annotation that should have step and line number information" });
+                embeddedStep.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice annotation that should have step and line number information" });
+
+                jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny<Guid>(), It.Is<TimelineRecord>(t => t.Issues.Where(i => i.Data.ContainsKey("stepNumber") && i.Data.ContainsKey("logFileLineNumber") && i.Type == IssueType.Error).Count() == 1)), Times.AtLeastOnce);
+                jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny<Guid>(), It.Is<TimelineRecord>(t => t.Issues.Where(i => i.Data.ContainsKey("stepNumber") && i.Data.ContainsKey("logFileLineNumber") && i.Type == IssueType.Warning).Count() == 1)), Times.AtLeastOnce);
+                jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny<Guid>(), It.Is<TimelineRecord>(t => t.Issues.Where(i => i.Data.ContainsKey("stepNumber") && i.Data.ContainsKey("logFileLineNumber") && i.Type == IssueType.Notice).Count() == 1)), Times.AtLeastOnce);
+            }
+        }
+
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]

src/Test/L0/Worker/JobExtensionL0.cs

@@ -25,6 +25,7 @@ namespace GitHub.Runner.Common.Tests.Worker
         private Mock<IPagingLogger> _logger;
         private Mock<IContainerOperationProvider> _containerProvider;
         private Mock<IDiagnosticLogManager> _diagnosticLogManager;
+        private Mock<IJobHookProvider> _jobHookProvider;
 
         private CancellationTokenSource _tokenSource;
         private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
@@ -40,6 +41,7 @@ namespace GitHub.Runner.Common.Tests.Worker
             _directoryManager = new Mock<IPipelineDirectoryManager>();
             _directoryManager.Setup(x => x.PrepareDirectory(It.IsAny<IExecutionContext>(), It.IsAny<Pipelines.WorkspaceOptions>()))
                              .Returns(new TrackingConfig() { PipelineDirectory = "runner", WorkspaceDirectory = "runner/runner" });
+            _jobHookProvider = new Mock<IJobHookProvider>();
 
             IActionRunner step1 = new ActionRunner();
             IActionRunner step2 = new ActionRunner();
@@ -111,7 +113,9 @@ namespace GitHub.Runner.Common.Tests.Worker
             hc.SetSingleton(_containerProvider.Object);
             hc.SetSingleton(_directoryManager.Object);
             hc.SetSingleton(_diagnosticLogManager.Object);
+            hc.SetSingleton(_jobHookProvider.Object);
             hc.EnqueueInstance<IPagingLogger>(_logger.Object); // JobExecutionContext
+            hc.EnqueueInstance<IPagingLogger>(_logger.Object); // job start hook
             hc.EnqueueInstance<IPagingLogger>(_logger.Object); // Initial Job
             hc.EnqueueInstance<IPagingLogger>(_logger.Object); // step1
             hc.EnqueueInstance<IPagingLogger>(_logger.Object); // step2
@@ -120,6 +124,7 @@ namespace GitHub.Runner.Common.Tests.Worker
             hc.EnqueueInstance<IPagingLogger>(_logger.Object); // step5
             hc.EnqueueInstance<IPagingLogger>(_logger.Object); // prepare1
             hc.EnqueueInstance<IPagingLogger>(_logger.Object); // prepare2
+            hc.EnqueueInstance<IPagingLogger>(_logger.Object); // job complete hook
 
             hc.EnqueueInstance<IActionRunner>(step1);
             hc.EnqueueInstance<IActionRunner>(step2);
@@ -348,5 +353,62 @@ namespace GitHub.Runner.Common.Tests.Worker
                 Assert.Equal(TaskResult.Succeeded, _jobEc.Result);
             }
         }
+
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public async Task EnsurePreAndPostHookStepsIfEnvExists()
+        {
+            Environment.SetEnvironmentVariable("ACTIONS_RUNNER_HOOK_JOB_STARTED", "/foo/bar");
+            Environment.SetEnvironmentVariable("ACTIONS_RUNNER_HOOK_JOB_COMPLETED", "/bar/foo");
+            using (TestHostContext hc = CreateTestContext())
+            {                
+                var jobExtension = new JobExtension();
+                jobExtension.Initialize(hc);
+
+                _actionManager.Setup(x => x.PrepareActionsAsync(It.IsAny<IExecutionContext>(), It.IsAny<IEnumerable<Pipelines.JobStep>>(), It.IsAny<Guid>()))
+                              .Returns(Task.FromResult(new PrepareResult(new List<JobExtensionRunner>(), new Dictionary<Guid, IActionRunner>())));
+
+                List<IStep> result = await jobExtension.InitializeJob(_jobEc, _message);
+
+                var trace = hc.GetTrace();
+
+                var hookStart = result.First() as JobExtensionRunner;
+
+                jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
+
+                Assert.Equal(Constants.Hooks.JobStartedStepName, hookStart.DisplayName);
+                Assert.Equal(Constants.Hooks.JobCompletedStepName, (_jobEc.PostJobSteps.Last() as JobExtensionRunner).DisplayName);                
+            }
+
+            Environment.SetEnvironmentVariable("ACTIONS_RUNNER_HOOK_JOB_STARTED", null);
+            Environment.SetEnvironmentVariable("ACTIONS_RUNNER_HOOK_JOB_COMPLETED", null);
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public void EnsureNoPreAndPostHookSteps()
+        {
+            using (TestHostContext hc = CreateTestContext())
+            {
+                var jobExtension = new JobExtension();
+                jobExtension.Initialize(hc);
+
+                _message.ActionsEnvironment = null;
+
+                _jobEc = new Runner.Worker.ExecutionContext {Result = TaskResult.Succeeded};
+                _jobEc.Initialize(hc);
+                _jobEc.InitializeJob(_message, _tokenSource.Token);
+
+                var x = _jobEc.JobSteps;
+
+                jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
+
+                Assert.Equal(TaskResult.Succeeded, _jobEc.Result);
+                Assert.Equal(0, _jobEc.PostJobSteps.Count);
+            }
+        }
     }
 }

src/Test/L0/Worker/StepsRunnerL0.cs

@@ -622,6 +622,40 @@ namespace GitHub.Runner.Common.Tests.Worker
                     _stepContext.SetOutcome("", stepContext.Object.ContextName, (stepContext.Object.Outcome ?? stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult());
                     _stepContext.SetConclusion("", stepContext.Object.ContextName, (stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult());
                 });
+
+            stepContext.Setup(x => x.UpdateGlobalStepsContext()).Callback(() =>
+            {
+                if (!string.IsNullOrEmpty(stepContext.Object.ContextName) && !stepContext.Object.ContextName.StartsWith("__", StringComparison.Ordinal))
+                {
+                    stepContext.Object.Global.StepsContext.SetOutcome(stepContext.Object.ScopeName, stepContext.Object.ContextName, (stepContext.Object.Outcome ?? stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult());
+                    stepContext.Object.Global.StepsContext.SetConclusion(stepContext.Object.ScopeName, stepContext.Object.ContextName, (stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult());
+                }
+            });
+            stepContext.Setup(x => x.ApplyContinueOnError(It.IsAny<TemplateToken>())).Callback((TemplateToken token) =>
+            {
+                if (stepContext.Object.Result != TaskResult.Failed)
+                {
+                    return;
+                }
+                var continueOnError = false;
+                try
+                {
+                    var templateEvaluator = stepContext.Object.ToPipelineTemplateEvaluator();
+                    continueOnError = templateEvaluator.EvaluateStepContinueOnError(token, stepContext.Object.ExpressionValues, stepContext.Object.ExpressionFunctions);
+                }
+                catch (Exception ex)
+                {
+                    stepContext.Object.Error("The step failed and an error occurred when attempting to determine whether to continue on error.");
+                    stepContext.Object.Error(ex);
+                }
+
+                if (continueOnError)
+                {
+                    stepContext.Object.Outcome = stepContext.Object.Result;
+                    stepContext.Object.Result = TaskResult.Succeeded;
+                }
+                stepContext.Object.UpdateGlobalStepsContext();
+            });
             var trace = hc.GetTrace();
             stepContext.Setup(x => x.Write(It.IsAny<string>(), It.IsAny<string>())).Callback((string tag, string message) => { trace.Info($"[{tag}]{message}"); });
             stepContext.Object.Result = result;

src/dev.sh

@@ -2,7 +2,7 @@
 
 ###############################################################################
 #  
-#  ./dev.sh build/layout/test/package [Debug/Release]
+#  ./dev.sh build/layout/test/package [Debug/Release] [linux-x64|linux-x86|linux-arm64|linux-arm|osx-x64|win-x64|win-x86] [use-broker]
 #
 ###############################################################################
 
@@ -11,6 +11,7 @@ set -e
 DEV_CMD=$1
 DEV_CONFIG=$2
 DEV_TARGET_RUNTIME=$3
+DEV_USE_BROKER=$4
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 LAYOUT_DIR="$SCRIPT_DIR/../_layout"
@@ -81,6 +82,13 @@ elif [[ "$CURRENT_PLATFORM" == 'darwin' ]]; then
     fi
 fi
 
+if [ -n "$DEV_USE_BROKER" ]; then
+    USE_BROKER='-p:USE_BROKER="true"'
+else
+    USE_BROKER=''
+fi
+
+
 function failed()
 {
     local error=${1:-Undefined error}
@@ -114,13 +122,13 @@ function heading()
 function build ()
 {
     heading "Building ..."
-    dotnet msbuild -t:Build -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" ./dir.proj || failed build
+    dotnet msbuild -t:Build -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" $USE_BROKER ./dir.proj || failed build
 }
 
 function layout ()
 {
     heading "Create layout ..."
-    dotnet msbuild -t:layout -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" ./dir.proj || failed build
+    dotnet msbuild -t:layout -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" $USE_BROKER ./dir.proj || failed build
 
     #change execution flag to allow running with sudo
     if [[ ("$CURRENT_PLATFORM" == "linux") || ("$CURRENT_PLATFORM" == "darwin") ]]; then
@@ -128,6 +136,7 @@ function layout ()
         chmod +x "${LAYOUT_DIR}/bin/Runner.Worker"
         chmod +x "${LAYOUT_DIR}/bin/Runner.PluginHost"
         chmod +x "${LAYOUT_DIR}/bin/installdependencies.sh"
+        chmod +x "${LAYOUT_DIR}/safe_sleep.sh"
     fi
 
     heading "Setup externals folder for $RUNTIME_ID runner's layout"

src/runnerversion

@@ -1 +1 @@
-2.288.0
+2.289.2