Implement termination grace period support for the runner

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.devcontainer/devcontainer.json

@@ -4,10 +4,10 @@
   "features": {
     "ghcr.io/devcontainers/features/docker-in-docker:1": {},
     "ghcr.io/devcontainers/features/dotnet": {
-      "version": "6.0.421"
+      "version": "8.0.408"
     },
     "ghcr.io/devcontainers/features/node:1": {
-      "version": "16"
+      "version": "20"
     },
     "ghcr.io/devcontainers/features/sshd:1": {
       "version": "latest"

.github/ISSUE_TEMPLATE/config.yml

@@ -7,7 +7,7 @@ contact_links:
     url: https://github.community/c/code-to-cloud/52
     about: If you have questions about GitHub Actions or need support writing workflows, please ask in the GitHub Community Support forum.
   - name: ✅ Feedback and suggestions for GitHub Actions
-    url: https://github.com/github/feedback/discussions/categories/actions-and-packages-feedback
+    url: https://github.com/github/feedback/discussions/categories/actions
     about: If you have feedback or suggestions about GitHub Actions, please open a discussion (or add to an existing one) in the GitHub Actions Feedback.  GitHub Actions Product Managers and Engineers monitor the feedback forum.
   - name: ‼️ GitHub Security Bug Bounty
     url: https://bounty.github.com/

.github/dependabot.yml

@@ -5,6 +5,11 @@ updates:
   schedule:
     interval: "daily" 
   target-branch: "main" 
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "daily" 
+  target-branch: "main" 
 - package-ecosystem: "nuget"
   directory: "/src"
   schedule:

.github/workflows/build.yml

@@ -50,7 +50,7 @@ jobs:
 
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Build runner layout
     - name: Build & Layout Release
@@ -69,7 +69,7 @@ jobs:
     - name: Package Release
       if: github.event_name != 'pull_request'
       run: |
-        ${{ matrix.devScript }} package Release
+        ${{ matrix.devScript }} package Release ${{ matrix.runtime }}
       working-directory: src
 
     # Upload runner package tar.gz/zip as artifact

.github/workflows/close-bugs-bot.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v8
+      - uses: actions/stale@v9
         with:
           close-issue-message: "This issue does not seem to be a problem with the runner application, it concerns the GitHub actions platform more generally. Could you please post your feedback on the [GitHub Community Support Forum](https://github.com/orgs/community/discussions/categories/actions) which is actively monitored. Using the forum ensures that we route your problem to the correct team. 😃"
           exempt-issue-labels: "keep"

.github/workflows/close-features-bot.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v8
+      - uses: actions/stale@v9
         with:
           close-issue-message: "Thank you for your interest in the runner application and taking the time to provide your valuable feedback. We kindly ask you to redirect this feedback to the [GitHub Community Support Forum](https://github.com/orgs/community/discussions/categories/actions-and-packages) which our team actively monitors and would be a better place to start a discussion for new feature requests in GitHub Actions. For more information on this policy please [read our contribution guidelines](https://github.com/actions/runner#contribute). 😃"
           exempt-issue-labels: "keep"

.github/workflows/codeql.yml

@@ -23,11 +23,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       # Override language selection by uncommenting this and choosing your languages
       # with:
       #   languages: go, javascript, csharp, python, cpp, java
@@ -38,4 +38,4 @@ jobs:
       working-directory: src
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/dotnet-upgrade.yml

@@ -15,7 +15,7 @@ jobs:
       DOTNET_CURRENT_MAJOR_MINOR_VERSION: ${{ steps.fetch_current_version.outputs.DOTNET_CURRENT_MAJOR_MINOR_VERSION }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get current major minor version
         id: fetch_current_version
         shell: bash
@@ -51,7 +51,7 @@ jobs:
         run: echo "::error links::feature/dotnet-sdk-upgrade${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }} https://github.com/actions/runner/tree/feature/dotnet-sdk-upgrade${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}::Branch feature/dotnetsdk-upgrade/${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }} already exists. Please take a look and delete that branch if you wish to recreate"
       - name: Create a warning annotation if no need to update
         if: ${{ steps.fetch_latest_version.outputs.SHOULD_UPDATE == 0 && steps.fetch_latest_version.outputs.BRANCH_EXISTS == 0 }}
-        run: echo "::warning ::Latest DotNet SDK patch is ${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}, and we are on ${{ steps.fetch_latest_version.outputs.DOTNET_CURRENT_MAJOR_MINOR_PATCH_VERSION }}. No need to update"
+        run: echo "::warning ::Latest DotNet SDK patch is ${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}, and we are on ${{ steps.fetch_current_version.outputs.DOTNET_CURRENT_MAJOR_MINOR_PATCH_VERSION }}. No need to update"
       - name: Update patch version
         if: ${{ steps.fetch_latest_version.outputs.SHOULD_UPDATE == 1 && steps.fetch_latest_version.outputs.BRANCH_EXISTS == 0 }}
         shell: bash
@@ -89,7 +89,7 @@ jobs:
     if: ${{ needs.dotnet-update.outputs.SHOULD_UPDATE == 1 && needs.dotnet-update.outputs.BRANCH_EXISTS == 0 }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: feature/dotnetsdk-upgrade/${{ needs.dotnet-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}
     - name: Create Pull Request

.github/workflows/lint.yml

@@ -1,24 +0,0 @@
-name: Lint
-
-on:
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-    name: Lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          # Ensure full list of changed files within `super-linter`
-          fetch-depth: 0
-      - name: Run linters
-        uses: github/super-linter@v4
-        env:
-          DEFAULT_BRANCH: ${{ github.base_ref }}
-          EDITORCONFIG_FILE_NAME: .editorconfig
-          LINTER_RULES_PATH: /src/
-          VALIDATE_ALL_CODEBASE: false
-          VALIDATE_CSHARP: true

.github/workflows/publish-image.yml

@@ -1,70 +0,0 @@
-name: Publish Runner Image
-
-on:
-  workflow_dispatch:
-    inputs:
-      runnerVersion:
-        type: string
-        description: Version of the runner being installed
-
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository_owner }}/actions-runner
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Compute image version
-        id: image
-        uses: actions/github-script@v6
-        env:
-          RUNNER_VERSION: ${{ github.event.inputs.runnerVersion }}
-        with:
-          script: |
-            const fs = require('fs');
-            const inputRunnerVersion = process.env.RUNNER_VERSION;
-            if (inputRunnerVersion) {
-              console.log(`Using input runner version ${inputRunnerVersion}`)
-              core.setOutput('version', inputRunnerVersion);
-              return
-            }
-            const runnerVersion = fs.readFileSync('${{ github.workspace }}/src/runnerversion', 'utf8').replace(/\n$/g, '')
-            console.log(`Using runner version ${runnerVersion}`)
-            core.setOutput('version', runnerVersion);
-
-      - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push Docker image
-        id: build-and-push
-        uses: docker/build-push-action@v3
-        with:
-          context: ./images
-          platforms: |
-            linux/amd64
-            linux/arm64
-          tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.image.outputs.version }}
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-          build-args: |
-            RUNNER_VERSION=${{ steps.image.outputs.version }}
-          push: true
-          labels: |
-            org.opencontainers.image.source=${{github.server_url}}/${{github.repository}}
-            org.opencontainers.image.description=https://github.com/actions/runner/releases/tag/v${{ steps.image.outputs.version }}
-            org.opencontainers.image.licenses=MIT

.github/workflows/release.yml

@@ -11,16 +11,15 @@ jobs:
     if: startsWith(github.ref, 'refs/heads/releases/') || github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Make sure ./releaseVersion match ./src/runnerversion
     # Query GitHub release ensure version is not used
     - name: Check version
-      uses: actions/github-script@0.3.0
+      uses: actions/github-script@v7.0.1
       with:
         github-token: ${{secrets.GITHUB_TOKEN}}
         script: |
-          const core = require('@actions/core')
           const fs = require('fs');
           const runnerVersion = fs.readFileSync('${{ github.workspace }}/src/runnerversion', 'utf8').replace(/\n$/g, '')
           const releaseVersion = fs.readFileSync('${{ github.workspace }}/releaseVersion', 'utf8').replace(/\n$/g, '')
@@ -30,7 +29,7 @@ jobs:
             return
           }
           try {
-            const release = await github.repos.getReleaseByTag({
+            const release = await github.rest.repos.getReleaseByTag({
               owner: '${{ github.event.repository.owner.name }}',
               repo: '${{ github.event.repository.name }}',
               tag: 'v' + runnerVersion
@@ -87,7 +86,7 @@ jobs:
 
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Build runner layout
     - name: Build & Layout Release
@@ -117,12 +116,11 @@ jobs:
       working-directory: _package
 
     # Upload runner package tar.gz/zip as artifact.
-    # Since each package name is unique, so we don't need to put ${{matrix}} info into artifact name
     - name: Publish Artifact
       if: github.event_name != 'pull_request'
       uses: actions/upload-artifact@v4
       with:
-        name: runner-packages
+        name: runner-packages-${{ matrix.runtime }}
         path: |
           _package
 
@@ -131,23 +129,52 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Download runner package tar.gz/zip produced by 'build' job
-    - name: Download Artifact
+    - name: Download Artifact (win-x64)
       uses: actions/download-artifact@v4
       with:
-        name: runner-packages
+        name: runner-packages-win-x64
+        path: ./
+    - name: Download Artifact (win-arm64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-win-arm64
+        path: ./
+    - name: Download Artifact (osx-x64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-osx-x64
+        path: ./
+    - name: Download Artifact (osx-arm64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-osx-arm64
+        path: ./
+    - name: Download Artifact (linux-x64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-linux-x64
+        path: ./
+    - name: Download Artifact (linux-arm)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-linux-arm
+        path: ./
+    - name: Download Artifact (linux-arm64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-linux-arm64
         path: ./
 
     # Create ReleaseNote file
     - name: Create ReleaseNote
       id: releaseNote
-      uses: actions/github-script@0.3.0
+      uses: actions/github-script@v7.0.1
       with:
         github-token: ${{secrets.GITHUB_TOKEN}}
         script: |
-          const core = require('@actions/core')
           const fs = require('fs');
           const runnerVersion = fs.readFileSync('${{ github.workspace }}/src/runnerversion', 'utf8').replace(/\n$/g, '')
           var releaseNote = fs.readFileSync('${{ github.workspace }}/releaseNote.md', 'utf8').replace(/<RUNNER_VERSION>/g, runnerVersion)
@@ -262,16 +289,18 @@ jobs:
     permissions:
       contents: read
       packages: write
+      id-token: write
+      attestations: write
     env:
       REGISTRY: ghcr.io
       IMAGE_NAME: ${{ github.repository_owner }}/actions-runner
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Compute image version
         id: image
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7.0.1
         with:
           script: |
             const fs = require('fs');
@@ -280,10 +309,10 @@ jobs:
             core.setOutput('version', runnerVersion);
 
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -291,7 +320,7 @@ jobs:
 
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: ./images
           platforms: |
@@ -307,3 +336,10 @@ jobs:
             org.opencontainers.image.source=${{github.server_url}}/${{github.repository}}
             org.opencontainers.image.description=https://github.com/actions/runner/releases/tag/v${{ steps.image.outputs.version }}
             org.opencontainers.image.licenses=MIT
+
+      - name: Generate attestation
+        uses: actions/attest-build-provenance@v2
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          subject-digest: ${{ steps.build-and-push.outputs.digest }}
+          push-to-registry: true

.github/workflows/stale-bot.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v8
+      - uses: actions/stale@v9
         with:
           stale-issue-message: "This issue is stale because it has been open 365 days with no activity. Remove stale label or comment or this will be closed in 15 days."
           close-issue-message: "This issue was closed because it has been stalled for 15 days with no activity."

.gitignore

@@ -26,4 +26,5 @@ _dotnetsdk
 TestResults
 TestLogs
 .DS_Store
+.mono
 **/*.DotSettings.user

docs/adrs/1751-runner-job-hooks.md

@@ -23,7 +23,7 @@ This feature is mainly intended for self hosted runner administrators.
   - `ACTIONS_RUNNER_HOOK_JOB_STARTED`
   - `ACTIONS_RUNNER_HOOK_JOB_COMPLETED`
 
-You can set these variables to the **absolute** path of a a `.sh` or `.ps1` file.
+You can set these variables to the **absolute** path of a `.sh` or `.ps1` file.
 
 We will execute `pwsh` (fallback to `powershell`) or `bash` (fallback to `sh`) as appropriate.
 - `.sh` files will execute with the args `-e {pathtofile}`

docs/checks/nodejs.md

@@ -4,9 +4,9 @@
 
 Make sure the built-in node.js has access to GitHub.com or GitHub Enterprise Server.
 
-The runner carries its own copy of node.js executable under `<runner_root>/externals/node16/`.
+The runner carries its own copy of node.js executable under `<runner_root>/externals/node20/`.
 
-All javascript base Actions will get executed by the built-in `node` at `<runner_root>/externals/node16/`.
+All javascript base Actions will get executed by the built-in `node` at `<runner_root>/externals/node20/`.
 
 > Not the `node` from `$PATH`
 

docs/contribute.md

@@ -1,6 +1,6 @@
 # Contributions
 
-We welcome contributions in the form of issues and pull requests. We view the contributions and the process as the same for github and external contributors.Please note the runner typically requires changes across the entire system and we aim for issues in the runner to be entirely self contained and fixable here. Therefore, we will primarily handle bug issues opened in this repo and we kindly request you to create all feature and enhancement requests on the [GitHub Feedback](https://github.com/community/community/discussions/categories/actions-and-packages) page. 
+We welcome contributions in the form of issues and pull requests. We view the contributions and the process as the same for github and external contributors. Please note the runner typically requires changes across the entire system and we aim for issues in the runner to be entirely self contained and fixable here. Therefore, we will primarily handle bug issues opened in this repo and we kindly request you to create all feature and enhancement requests on the [GitHub Feedback](https://github.com/community/community/discussions/categories/actions-and-packages) page. 
 
 > IMPORTANT: Building your own runner is critical for the dev inner loop process when contributing changes.  However, only runners built and distributed by GitHub (releases) are supported in production.  Be aware that workflows and orchestrations run service side with the runner being a remote process to run steps.  For that reason, the service can pull the runner forward so customizations can be lost.
 

images/Dockerfile

@@ -1,12 +1,12 @@
 # Source: https://github.com/dotnet/dotnet-docker
-FROM mcr.microsoft.com/dotnet/runtime-deps:6.0-jammy as build
+FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-jammy AS build
 
 ARG TARGETOS
 ARG TARGETARCH
 ARG RUNNER_VERSION
-ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.1
+ARG RUNNER_CONTAINER_HOOKS_VERSION=0.7.0
-ARG DOCKER_VERSION=27.1.1
+ARG DOCKER_VERSION=28.0.1
-ARG BUILDX_VERSION=0.16.2
+ARG BUILDX_VERSION=0.21.2
 
 RUN apt update -y && apt install curl unzip -y
 
@@ -32,7 +32,7 @@ RUN export RUNNER_ARCH=${TARGETARCH} \
         "https://github.com/docker/buildx/releases/download/v${BUILDX_VERSION}/buildx-v${BUILDX_VERSION}.linux-${TARGETARCH}" \
     && chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx
 
-FROM mcr.microsoft.com/dotnet/runtime-deps:6.0-jammy
+FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-jammy
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV RUNNER_MANUALLY_TRAP_SIG=1
@@ -47,7 +47,8 @@ RUN apt update -y \
 # Configure git-core/ppa based on guidance here:  https://git-scm.com/download/linux
 RUN add-apt-repository ppa:git-core/ppa \
     && apt update -y \
-    && apt install -y --no-install-recommends git
+    && apt install -y git \
+    && rm -rf /var/lib/apt/lists/*
 
 RUN adduser --disabled-password --gecos "" --uid 1001 runner \
     && groupadd docker --gid 123 \

releaseNote.md

@@ -1,17 +1,36 @@
 ## What's Changed
+* Bump docker/login-action from 2 to 3 by @dependabot in https://github.com/actions/runner/pull/3673
+* Bump actions/stale from 8 to 9 by @dependabot in https://github.com/actions/runner/pull/3554
+* Bump docker/build-push-action from 3 to 6 by @dependabot in https://github.com/actions/runner/pull/3674
+* update node version from 20.18.0 -> 20.18.2 by @aiqiaoy in https://github.com/actions/runner/pull/3682
+* Pass BillingOwnerId through Acquire/Complete calls by @luketomlinson in https://github.com/actions/runner/pull/3689
+* Do not retry CompleteJobAsync for known non-retryable errors by @ericsciple in https://github.com/actions/runner/pull/3696
+* Update dotnet sdk to latest version @8.0.406 by @github-actions in https://github.com/actions/runner/pull/3712
+* Update Dockerfile with new docker and buildx versions by @thboop in https://github.com/actions/runner/pull/3680
+* chore: remove redundant words by @finaltrip in https://github.com/actions/runner/pull/3705
+* fix: actions feedback link is incorrect by @Yaminyam in https://github.com/actions/runner/pull/3165
+* Bump actions/github-script from 0.3.0 to 7.0.1 by @dependabot in https://github.com/actions/runner/pull/3557
+* Docker container provenance by @paveliak in https://github.com/actions/runner/pull/3736
+* Add request-id to http eventsource trace. by @TingluoHuang in https://github.com/actions/runner/pull/3740
+* Update Bocker and Buildx version to mitigate images scanners alerts by @Blizter in https://github.com/actions/runner/pull/3750
+* Fix typo, add invariant culture to timestamp for workflow log reporting by @GhadimiR in https://github.com/actions/runner/pull/3749
+* Create vssconnection to actions service when URL provided. by @TingluoHuang in https://github.com/actions/runner/pull/3751
+* Housekeeping: Update npm packages and node version by @thboop in https://github.com/actions/runner/pull/3752
+* Improve the out-of-date warning message. by @tecimovic in https://github.com/actions/runner/pull/3595
+* Update dotnet sdk to latest version @8.0.407 by @github-actions in https://github.com/actions/runner/pull/3753
+* Exit hosted runner cleanly during deprovisioning. by @TingluoHuang in https://github.com/actions/runner/pull/3755
+* Send annotation title to run-service. by @TingluoHuang in https://github.com/actions/runner/pull/3757
+* Allow server enforce runner settings. by @TingluoHuang in https://github.com/actions/runner/pull/3758
+* Support refresh runner configs with pipelines service. by @TingluoHuang in https://github.com/actions/runner/pull/3706
 
-- Adding Snapshot additional mapping tokens https://github.com/actions/runner/pull/3468
+## New Contributors
-- Create launch httpclient using the right handler and setting https://github.com/actions/runner/pull/3476
+* @finaltrip made their first contribution in https://github.com/actions/runner/pull/3705
-- Fix missing default user-agent for jitconfig runner https://github.com/actions/runner/pull/3473
+* @Yaminyam made their first contribution in https://github.com/actions/runner/pull/3165
-- Cleanup back-compat code for interpreting Run Service status codes https://github.com/actions/runner/pull/3456
+* @Blizter made their first contribution in https://github.com/actions/runner/pull/3750
-- Add runner or worker to the useragent https://github.com/actions/runner/pull/3457
+* @GhadimiR made their first contribution in https://github.com/actions/runner/pull/3749
-- Handle Error Body in Responses from Broker https://github.com/actions/runner/pull/3454
+* @tecimovic made their first contribution in https://github.com/actions/runner/pull/3595
-- Fix issues for composite actions (Run Service flow) https://github.com/actions/runner/pull/3446
-- Trace GitHub RequestId to log https://github.com/actions/runner/pull/3442
-- Add `jq`, `git`, `unzip` and `curl` to default packages installed https://github.com/actions/runner/pull/3056
-- Add pid to user-agent and session owner https://github.com/actions/runner/pull/3432
 
-**Full Changelog**: https://github.com/actions/runner/compare/v2.319.1...v2.320.0
+**Full Changelog**: https://github.com/actions/runner/compare/v2.322.0...v2.323.0
 
 _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet.
 To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository.
@@ -33,9 +52,7 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem ;
 [System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD\actions-runner-win-x64-<RUNNER_VERSION>.zip", "$PWD")
 ```
 
-## [Pre-release] Windows arm64
+## Windows arm64
-
-**Warning:** Windows arm64 runners are currently in preview status and use [unofficial versions of nodejs](https://unofficial-builds.nodejs.org/). They are not intended for production workflows.
 
 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.
 

src/Directory.Build.props

@@ -57,4 +57,13 @@
   <PropertyGroup>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>
+
+  <PropertyGroup>
+    <!-- Enable NuGet package auditing -->
+    <NuGetAudit>true</NuGetAudit>
+    <!-- Audit direct and transitive packages -->
+    <NuGetAuditMode>all</NuGetAuditMode>
+    <!-- Report low, moderate, high and critical advisories -->
+    <NuGetAuditLevel>moderate</NuGetAuditLevel>
+  </PropertyGroup>
 </Project>

src/Misc/expressionFunc/hashFiles/package.json

@@ -11,7 +11,6 @@
     "pack": "ncc build -o ../../layoutbin/hashFiles",
     "all": "npm run format && npm run lint && npm run build && npm run pack",
     "prepare": "cd ../../../../ && husky install"
-
   },
   "repository": {
     "type": "git",
@@ -43,9 +42,9 @@
     "eslint": "^8.47.0",
     "eslint-plugin-github": "^4.10.0",
     "eslint-plugin-prettier": "^5.0.0",
-    "prettier": "^3.0.3",
-    "typescript": "^5.2.2",
     "husky": "^8.0.3",
-    "lint-staged": "^14.0.0"
+    "lint-staged": "^15.5.0",
+    "prettier": "^3.0.3",
+    "typescript": "^5.2.2"
   }
-}
+}

src/Misc/externals.sh

@@ -3,13 +3,10 @@ PACKAGERUNTIME=$1
 PRECACHE=$2
 
 NODE_URL=https://nodejs.org/dist
-UNOFFICIAL_NODE_URL=https://unofficial-builds.nodejs.org/download/release
 NODE_ALPINE_URL=https://github.com/actions/alpine_nodejs/releases/download
 # When you update Node versions you must also create a new release of alpine_nodejs at that updated version.
 # Follow the instructions here: https://github.com/actions/alpine_nodejs?tab=readme-ov-file#getting-started
-NODE16_VERSION="16.20.2"
+NODE20_VERSION="20.19.0"
-NODE20_VERSION="20.13.1"
-NODE16_UNOFFICIAL_VERSION="16.20.0" # used only for win-arm64, remove node16 unofficial version when official version is available
 
 get_abs_path() {
   # exploits the fact that pwd will print abs path when no args
@@ -140,8 +137,6 @@ function acquireExternalTool() {
 
 # Download the external tools only for Windows.
 if [[ "$PACKAGERUNTIME" == "win-x64" || "$PACKAGERUNTIME" == "win-x86" ]]; then
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/$PACKAGERUNTIME/node.exe" node16/bin
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/$PACKAGERUNTIME/node.lib" node16/bin
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/$PACKAGERUNTIME/node.exe" node20/bin
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/$PACKAGERUNTIME/node.lib" node20/bin
     if [[ "$PRECACHE" != "" ]]; then
@@ -152,8 +147,6 @@ fi
 # Download the external tools only for Windows.
 if [[ "$PACKAGERUNTIME" == "win-arm64" ]]; then
     # todo: replace these with official release when available
-    acquireExternalTool "$UNOFFICIAL_NODE_URL/v${NODE16_UNOFFICIAL_VERSION}/$PACKAGERUNTIME/node.exe" node16/bin
-    acquireExternalTool "$UNOFFICIAL_NODE_URL/v${NODE16_UNOFFICIAL_VERSION}/$PACKAGERUNTIME/node.lib" node16/bin
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/$PACKAGERUNTIME/node.exe" node20/bin
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/$PACKAGERUNTIME/node.lib" node20/bin
     if [[ "$PRECACHE" != "" ]]; then
@@ -163,30 +156,24 @@ fi
 
 # Download the external tools only for OSX.
 if [[ "$PACKAGERUNTIME" == "osx-x64" ]]; then
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-darwin-x64.tar.gz" node16 fix_nested_dir
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-darwin-x64.tar.gz" node20 fix_nested_dir
 fi
 
 if [[ "$PACKAGERUNTIME" == "osx-arm64" ]]; then
     # node.js v12 doesn't support macOS on arm64.
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-darwin-arm64.tar.gz" node16 fix_nested_dir
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-darwin-arm64.tar.gz" node20 fix_nested_dir
 fi
 
 # Download the external tools for Linux PACKAGERUNTIMEs.
 if [[ "$PACKAGERUNTIME" == "linux-x64" ]]; then
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-linux-x64.tar.gz" node16 fix_nested_dir
-    acquireExternalTool "$NODE_ALPINE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-alpine-x64.tar.gz" node16_alpine
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-linux-x64.tar.gz" node20 fix_nested_dir
     acquireExternalTool "$NODE_ALPINE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-alpine-x64.tar.gz" node20_alpine
 fi
 
 if [[ "$PACKAGERUNTIME" == "linux-arm64" ]]; then
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-linux-arm64.tar.gz" node16 fix_nested_dir
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-linux-arm64.tar.gz" node20 fix_nested_dir
 fi
 
 if [[ "$PACKAGERUNTIME" == "linux-arm" ]]; then
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-linux-armv7l.tar.gz" node16 fix_nested_dir
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-linux-armv7l.tar.gz" node20 fix_nested_dir
 fi

src/Misc/layoutbin/runsvc.sh

@@ -10,7 +10,7 @@ if [ -f ".path" ]; then
     echo ".path=${PATH}"
 fi
 
-nodever=${GITHUB_ACTIONS_RUNNER_FORCED_NODE_VERSION:-node16}
+nodever="node20"
 
 # insert anything to setup env when running as a service
 # run the host process which keep the listener alive

src/Misc/layoutbin/update.sh.template

@@ -135,12 +135,17 @@ if [[ "$currentplatform" == 'darwin'  && restartinteractiverunner -eq 0 ]]; then
     then
         # inspect the open file handles to find the node process
         # we can't actually inspect the process using ps because it uses relative paths and doesn't follow symlinks
-        nodever="node16"
+        nodever="node20"
         path=$(lsof -a -g "$procgroup" -F n | grep $nodever/bin/node | grep externals | tail -1 | cut -c2-)
-        if [[ $? -ne 0 || -z "$path" ]] # Fallback if RunnerService.js was started with node12
+        if [[ $? -ne 0 || -z "$path" ]] # Fallback if RunnerService.js was started with node16
         then
-            nodever="node12"
+            nodever="node16"
             path=$(lsof -a -g "$procgroup" -F n | grep $nodever/bin/node | grep externals | tail -1 | cut -c2-)
+            if [[ $? -ne 0 || -z "$path" ]] # Fallback if RunnerService.js was started with node12
+            then
+                nodever="node12"
+                path=$(lsof -a -g "$procgroup" -F n | grep $nodever/bin/node | grep externals | tail -1 | cut -c2-)
+            fi
         fi
         if [[ $? -eq 0 && -n "$path" ]]
         then
@@ -178,6 +183,19 @@ if [[ "$currentplatform" == 'darwin'  && restartinteractiverunner -eq 0 ]]; then
     fi
 fi
 
+# update runsvc.sh
+if [ -f "$rootfolder/runsvc.sh" ]
+then
+    date "+[%F %T-%4N] Update runsvc.sh" >> "$logfile" 2>&1
+    cat "$rootfolder/bin/runsvc.sh" > "$rootfolder/runsvc.sh"
+    if [ $? -ne 0 ]
+    then
+        date "+[%F %T-%4N] Can't update $rootfolder/runsvc.sh using $rootfolder/bin/runsvc.sh" >> "$logfile" 2>&1
+        mv -fv "$logfile" "$logfile.failed"
+        exit 1
+    fi
+fi
+
 date "+[%F %T-%4N] Update succeed" >> "$logfile"
 
 touch update.finished

src/Runner.Common/AuthMigration.cs

@@ -0,0 +1,13 @@
+using System;
+
+namespace GitHub.Runner.Common
+{
+    public class AuthMigrationEventArgs : EventArgs
+    {
+        public AuthMigrationEventArgs(string trace)
+        {
+            Trace = trace;
+        }
+        public string Trace { get; private set; }
+    }
+}

src/Runner.Common/BrokerServer.cs

@@ -7,6 +7,7 @@ using GitHub.DistributedTask.Pipelines;
 using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
+using GitHub.Services.WebApi;
 using Sdk.RSWebApi.Contracts;
 using Sdk.WebApi.WebApi.RawClient;
 
@@ -36,6 +37,7 @@ namespace GitHub.Runner.Common
 
         public async Task ConnectAsync(Uri serverUri, VssCredentials credentials)
         {
+            Trace.Entering();
             _brokerUri = serverUri;
 
             _connection = VssUtil.CreateRawConnection(serverUri, credentials);
@@ -87,12 +89,17 @@ namespace GitHub.Runner.Common
 
         public Task ForceRefreshConnection(VssCredentials credentials)
         {
-            return ConnectAsync(_brokerUri, credentials);
+            if (!string.IsNullOrEmpty(_brokerUri?.AbsoluteUri))
+            {
+                return ConnectAsync(_brokerUri, credentials);
+            }
+
+            return Task.CompletedTask;
         }
 
         public bool ShouldRetryException(Exception ex)
         {
-            if (ex is AccessDeniedException ade)
+            if (ex is AccessDeniedException || ex is RunnerNotFoundException || ex is HostedRunnerDeprovisionedException)
             {
                 return false;
             }

src/Runner.Common/ConfigurationStore.cs

@@ -119,8 +119,11 @@ namespace GitHub.Runner.Common
         CredentialData GetCredentials();
         CredentialData GetMigratedCredentials();
         RunnerSettings GetSettings();
+        RunnerSettings GetMigratedSettings();
         void SaveCredential(CredentialData credential);
+        void SaveMigratedCredential(CredentialData credential);
         void SaveSettings(RunnerSettings settings);
+        void SaveMigratedSettings(RunnerSettings settings);
         void DeleteCredential();
         void DeleteMigratedCredential();
         void DeleteSettings();
@@ -130,6 +133,7 @@ namespace GitHub.Runner.Common
     {
         private string _binPath;
         private string _configFilePath;
+        private string _migratedConfigFilePath;
         private string _credFilePath;
         private string _migratedCredFilePath;
         private string _serviceConfigFilePath;
@@ -137,6 +141,7 @@ namespace GitHub.Runner.Common
         private CredentialData _creds;
         private CredentialData _migratedCreds;
         private RunnerSettings _settings;
+        private RunnerSettings _migratedSettings;
 
         public override void Initialize(IHostContext hostContext)
         {
@@ -154,6 +159,9 @@ namespace GitHub.Runner.Common
             _configFilePath = hostContext.GetConfigFile(WellKnownConfigFile.Runner);
             Trace.Info("ConfigFilePath: {0}", _configFilePath);
 
+            _migratedConfigFilePath = hostContext.GetConfigFile(WellKnownConfigFile.MigratedRunner);
+            Trace.Info("MigratedConfigFilePath: {0}", _migratedConfigFilePath);
+
             _credFilePath = hostContext.GetConfigFile(WellKnownConfigFile.Credentials);
             Trace.Info("CredFilePath: {0}", _credFilePath);
 
@@ -169,7 +177,7 @@ namespace GitHub.Runner.Common
         public bool HasCredentials()
         {
             Trace.Info("HasCredentials()");
-            bool credsStored = (new FileInfo(_credFilePath)).Exists || (new FileInfo(_migratedCredFilePath)).Exists;
+            bool credsStored = new FileInfo(_credFilePath).Exists || new FileInfo(_migratedCredFilePath).Exists;
             Trace.Info("stored {0}", credsStored);
             return credsStored;
         }
@@ -177,7 +185,7 @@ namespace GitHub.Runner.Common
         public bool IsConfigured()
         {
             Trace.Info("IsConfigured()");
-            bool configured = new FileInfo(_configFilePath).Exists;
+            bool configured = new FileInfo(_configFilePath).Exists || new FileInfo(_migratedConfigFilePath).Exists;
             Trace.Info("IsConfigured: {0}", configured);
             return configured;
         }
@@ -185,7 +193,7 @@ namespace GitHub.Runner.Common
         public bool IsServiceConfigured()
         {
             Trace.Info("IsServiceConfigured()");
-            bool serviceConfigured = (new FileInfo(_serviceConfigFilePath)).Exists;
+            bool serviceConfigured = new FileInfo(_serviceConfigFilePath).Exists;
             Trace.Info($"IsServiceConfigured: {serviceConfigured}");
             return serviceConfigured;
         }
@@ -229,6 +237,25 @@ namespace GitHub.Runner.Common
             return _settings;
         }
 
+        public RunnerSettings GetMigratedSettings()
+        {
+            if (_migratedSettings == null)
+            {
+                RunnerSettings configuredSettings = null;
+                if (File.Exists(_migratedConfigFilePath))
+                {
+                    string json = File.ReadAllText(_migratedConfigFilePath, Encoding.UTF8);
+                    Trace.Info($"Read migrated setting file: {json.Length} chars");
+                    configuredSettings = StringUtil.ConvertFromJson<RunnerSettings>(json);
+                }
+
+                ArgUtil.NotNull(configuredSettings, nameof(configuredSettings));
+                _migratedSettings = configuredSettings;
+            }
+
+            return _migratedSettings;
+        }
+
         public void SaveCredential(CredentialData credential)
         {
             Trace.Info("Saving {0} credential @ {1}", credential.Scheme, _credFilePath);
@@ -244,6 +271,21 @@ namespace GitHub.Runner.Common
             File.SetAttributes(_credFilePath, File.GetAttributes(_credFilePath) | FileAttributes.Hidden);
         }
 
+        public void SaveMigratedCredential(CredentialData credential)
+        {
+            Trace.Info("Saving {0} migrated credential @ {1}", credential.Scheme, _migratedCredFilePath);
+            if (File.Exists(_migratedCredFilePath))
+            {
+                // Delete existing credential file first, since the file is hidden and not able to overwrite.
+                Trace.Info("Delete exist runner migrated credential file.");
+                IOUtil.DeleteFile(_migratedCredFilePath);
+            }
+
+            IOUtil.SaveObject(credential, _migratedCredFilePath);
+            Trace.Info("Migrated Credentials Saved.");
+            File.SetAttributes(_migratedCredFilePath, File.GetAttributes(_migratedCredFilePath) | FileAttributes.Hidden);
+        }
+
         public void SaveSettings(RunnerSettings settings)
         {
             Trace.Info("Saving runner settings.");
@@ -259,6 +301,21 @@ namespace GitHub.Runner.Common
             File.SetAttributes(_configFilePath, File.GetAttributes(_configFilePath) | FileAttributes.Hidden);
         }
 
+        public void SaveMigratedSettings(RunnerSettings settings)
+        {
+            Trace.Info("Saving runner migrated settings");
+            if (File.Exists(_migratedConfigFilePath))
+            {
+                // Delete existing settings file first, since the file is hidden and not able to overwrite.
+                Trace.Info("Delete exist runner migrated settings file.");
+                IOUtil.DeleteFile(_migratedConfigFilePath);
+            }
+
+            IOUtil.SaveObject(settings, _migratedConfigFilePath);
+            Trace.Info("Migrated Settings Saved.");
+            File.SetAttributes(_migratedConfigFilePath, File.GetAttributes(_migratedConfigFilePath) | FileAttributes.Hidden);
+        }
+
         public void DeleteCredential()
         {
             IOUtil.Delete(_credFilePath, default(CancellationToken));
@@ -273,6 +330,12 @@ namespace GitHub.Runner.Common
         public void DeleteSettings()
         {
             IOUtil.Delete(_configFilePath, default(CancellationToken));
+            IOUtil.Delete(_migratedConfigFilePath, default(CancellationToken));
+        }
+
+        public void DeleteMigratedSettings()
+        {
+            IOUtil.Delete(_migratedConfigFilePath, default(CancellationToken));
         }
     }
 }

src/Runner.Common/Constants.cs

@@ -18,6 +18,7 @@ namespace GitHub.Runner.Common
     public enum WellKnownConfigFile
     {
         Runner,
+        MigratedRunner,
         Credentials,
         MigratedCredentials,
         RSACredentials,
@@ -159,7 +160,6 @@ namespace GitHub.Runner.Common
             public static class Features
             {
                 public static readonly string DiskSpaceWarning = "runner.diskspace.warning";
-                public static readonly string Node16Warning = "DistributedTask.AddWarningToNode16Action";
                 public static readonly string LogTemplateErrorsAsDebugMessages = "DistributedTask.LogTemplateErrorsAsDebugMessages";
                 public static readonly string UseContainerPathForTemplate = "DistributedTask.UseContainerPathForTemplate";
                 public static readonly string AllowRunnerContainerHooks = "DistributedTask.AllowRunnerContainerHooks";
@@ -176,14 +176,6 @@ namespace GitHub.Runner.Common
             public static readonly string UnsupportedStopCommandTokenDisabled = "You cannot use a endToken that is an empty string, the string 'pause-logging', or another workflow command. For more information see: https://docs.github.com/actions/learn-github-actions/workflow-commands-for-github-actions#example-stopping-and-starting-workflow-commands or opt into insecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS` environment variable to `true`.";
             public static readonly string UnsupportedSummarySize = "$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of {0}k, got {1}k. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary";
             public static readonly string SummaryUploadError = "$GITHUB_STEP_SUMMARY upload aborted, an error occurred when uploading the summary. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary";
-            public static readonly string DetectedNodeAfterEndOfLifeMessage = "Node.js 16 actions are deprecated. Please update the following actions to use Node.js 20: {0}. For more information see: https://github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/.";
-            public static readonly string DeprecatedNodeDetectedAfterEndOfLifeActions = "DeprecatedNodeActionsMessageWarnings";
-            public static readonly string DeprecatedNodeVersion = "node16";
-            public static readonly string EnforcedNode12DetectedAfterEndOfLife = "The following actions uses node12 which is deprecated and will be forced to run on node16: {0}. For more info: https://github.blog/changelog/2023-06-13-github-actions-all-actions-will-run-on-node16-instead-of-node12-by-default/";
-            public static readonly string EnforcedNode12DetectedAfterEndOfLifeEnvVariable = "Node16ForceActionsWarnings";
-            public static readonly string EnforcedNode16DetectedAfterEndOfLife = "The following actions use a deprecated Node.js version and will be forced to run on node20: {0}. For more info: https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/";
-            public static readonly string EnforcedNode16DetectedAfterEndOfLifeEnvVariable = "Node20ForceActionsWarnings";
-
         }
 
         public static class RunnerEvent
@@ -254,20 +246,18 @@ namespace GitHub.Runner.Common
                 public static readonly string RequireJobContainer = "ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER";
                 public static readonly string RunnerDebug = "ACTIONS_RUNNER_DEBUG";
                 public static readonly string StepDebug = "ACTIONS_STEP_DEBUG";
-                public static readonly string AllowActionsUseUnsecureNodeVersion = "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION";
-                public static readonly string ManualForceActionsToNode20 = "FORCE_JAVASCRIPT_ACTIONS_TO_NODE20";
             }
 
             public static class Agent
             {
                 public static readonly string ToolsDirectory = "agent.ToolsDirectory";
 
-                // Set this env var to "node12" to downgrade the node version for internal functions (e.g hashfiles). This does NOT affect the version of node actions.
+                // Set this env var to "nodeXY" to downgrade the node version for internal functions (e.g hashfiles). This does NOT affect the version of node actions.
                 public static readonly string ForcedInternalNodeVersion = "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION";
                 public static readonly string ForcedActionsNodeVersion = "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION";
                 public static readonly string PrintLogToStdout = "ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT";
                 public static readonly string ActionArchiveCacheDirectory = "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE";
-                public static readonly string ManualForceActionsToNode20 = "FORCE_JAVASCRIPT_ACTIONS_TO_NODE20";
+                public static readonly string ActionsTerminationGracePeriodSeconds = "ACTIONS_RUNNER_TERMINATION_GRACE_PERIOD_SECONDS";
             }
 
             public static class System
@@ -280,10 +270,6 @@ namespace GitHub.Runner.Common
                 public static readonly string PhaseDisplayName = "system.phaseDisplayName";
                 public static readonly string JobRequestType = "system.jobRequestType";
                 public static readonly string OrchestrationId = "system.orchestrationId";
-                public static readonly string TestDotNet8Compatibility = "system.testDotNet8Compatibility";
-                public static readonly string DotNet8CompatibilityOutputLength = "system.dotNet8CompatibilityOutputLength";
-                public static readonly string DotNet8CompatibilityOutputPattern = "system.dotNet8CompatibilityOutputPattern";
-                public static readonly string DotNet8CompatibilityWarning = "system.dotNet8CompatibilityWarning";
             }
         }
 

src/Runner.Common/HostContext.cs

@@ -34,9 +34,14 @@ namespace GitHub.Runner.Common
         T GetService<T>() where T : class, IRunnerService;
         void SetDefaultCulture(string name);
         event EventHandler Unloading;
-        void ShutdownRunner(ShutdownReason reason);
+        void ShutdownRunner(ShutdownReason reason, TimeSpan delay = default);
         void WritePerfCounter(string counter);
         void LoadDefaultUserAgents();
+
+        bool AllowAuthMigration { get; }
+        void EnableAuthMigration(string trace);
+        void DeferAuthMigration(TimeSpan deferred, string trace);
+        event EventHandler<AuthMigrationEventArgs> AuthMigrationChanged;
     }
 
     public enum StartupType
@@ -69,13 +74,24 @@ namespace GitHub.Runner.Common
         private string _perfFile;
         private RunnerWebProxy _webProxy = new();
         private string _hostType = string.Empty;
+        private ShutdownReason _shutdownReason = ShutdownReason.UserCancelled;
+        private int _shutdownReasonSet = 0;
+
+        // disable auth migration by default
+        private readonly ManualResetEventSlim _allowAuthMigration = new ManualResetEventSlim(false);
+        private DateTime _deferredAuthMigrationTime = DateTime.MaxValue;
+        private readonly object _authMigrationLock = new object();
+        private CancellationTokenSource _authMigrationAutoReenableTaskCancellationTokenSource = new();
+        private Task _authMigrationAutoReenableTask;
 
         public event EventHandler Unloading;
+        public event EventHandler<AuthMigrationEventArgs> AuthMigrationChanged;
         public CancellationToken RunnerShutdownToken => _runnerShutdownTokenSource.Token;
-        public ShutdownReason RunnerShutdownReason { get; private set; }
+        public ShutdownReason RunnerShutdownReason => _shutdownReason;
         public ISecretMasker SecretMasker => _secretMasker;
         public List<ProductInfoHeaderValue> UserAgents => _userAgents;
         public RunnerWebProxy WebProxy => _webProxy;
+        public bool AllowAuthMigration => _allowAuthMigration.IsSet;
         public HostContext(string hostType, string logFile = null)
         {
             // Validate args.
@@ -207,6 +223,71 @@ namespace GitHub.Runner.Common
             LoadDefaultUserAgents();
         }
 
+        // marked as internal for testing
+        internal async Task AuthMigrationAuthReenableAsync(TimeSpan refreshInterval, CancellationToken token)
+        {
+            try
+            {
+                while (!token.IsCancellationRequested)
+                {
+                    _trace.Verbose($"Auth migration defer timer is set to expire at {_deferredAuthMigrationTime.ToString("O")}. AllowAuthMigration: {_allowAuthMigration.IsSet}.");
+                    await Task.Delay(refreshInterval, token);
+                    if (!_allowAuthMigration.IsSet && DateTime.UtcNow > _deferredAuthMigrationTime)
+                    {
+                        _trace.Info($"Auth migration defer timer expired. Allowing auth migration.");
+                        EnableAuthMigration("Auth migration defer timer expired.");
+                    }
+                }
+            }
+            catch (TaskCanceledException)
+            {
+                // Task was cancelled, exit the loop.
+            }
+            catch (Exception ex)
+            {
+                _trace.Info("Error in auth migration reenable task.");
+                _trace.Error(ex);
+            }
+        }
+
+        public void EnableAuthMigration(string trace)
+        {
+            _allowAuthMigration.Set();
+
+            lock (_authMigrationLock)
+            {
+                if (_authMigrationAutoReenableTask == null)
+                {
+                    var refreshIntervalInMS = 60 * 1000;
+#if DEBUG
+                    // For L0, we will refresh faster
+                    if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL")))
+                    {
+                        refreshIntervalInMS = int.Parse(Environment.GetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL"));
+                    }
+#endif
+                    _authMigrationAutoReenableTask = AuthMigrationAuthReenableAsync(TimeSpan.FromMilliseconds(refreshIntervalInMS), _authMigrationAutoReenableTaskCancellationTokenSource.Token);
+                }
+            }
+
+            _trace.Info($"Enable auth migration at {DateTime.UtcNow.ToString("O")}.");
+            AuthMigrationChanged?.Invoke(this, new AuthMigrationEventArgs(trace));
+        }
+
+        public void DeferAuthMigration(TimeSpan deferred, string trace)
+        {
+            _allowAuthMigration.Reset();
+
+            // defer migration for a while
+            lock (_authMigrationLock)
+            {
+                _deferredAuthMigrationTime = DateTime.UtcNow.Add(deferred);
+            }
+
+            _trace.Info($"Disabled auth migration until {_deferredAuthMigrationTime.ToString("O")}.");
+            AuthMigrationChanged?.Invoke(this, new AuthMigrationEventArgs(trace));
+        }
+
         public void LoadDefaultUserAgents()
         {
             if (string.IsNullOrEmpty(WebProxy.HttpProxyAddress) && string.IsNullOrEmpty(WebProxy.HttpsProxyAddress))
@@ -343,6 +424,12 @@ namespace GitHub.Runner.Common
                         ".runner");
                     break;
 
+                case WellKnownConfigFile.MigratedRunner:
+                    path = Path.Combine(
+                        GetDirectory(WellKnownDirectory.Root),
+                        ".runner_migrated");
+                    break;
+
                 case WellKnownConfigFile.Credentials:
                     path = Path.Combine(
                         GetDirectory(WellKnownDirectory.Root),
@@ -488,12 +575,28 @@ namespace GitHub.Runner.Common
         }
 
 
-        public void ShutdownRunner(ShutdownReason reason)
+        public void ShutdownRunner(ShutdownReason reason, TimeSpan delay = default)
         {
             ArgUtil.NotNull(reason, nameof(reason));
-            _trace.Info($"Runner will be shutdown for {reason.ToString()}");
+            _trace.Info($"Runner will be shutdown for {reason.ToString()} after {delay.TotalSeconds} seconds.");
-            RunnerShutdownReason = reason;
+            if (Interlocked.CompareExchange(ref _shutdownReasonSet, 1, 0) == 0)
-            _runnerShutdownTokenSource.Cancel();
+            {
+                // Set the shutdown reason only if it hasn't been set before.
+                _shutdownReason = reason;
+            }
+            else
+            {
+                _trace.Verbose($"Runner shutdown reason already set to {_shutdownReason.ToString()}.");
+            }
+
+            if (delay.TotalSeconds == 0)
+            {
+                _runnerShutdownTokenSource.Cancel();
+            }
+            else
+            {
+                _runnerShutdownTokenSource.CancelAfter(delay);
+            }
         }
 
         public override void Dispose()
@@ -543,6 +646,18 @@ namespace GitHub.Runner.Common
                     _loadContext.Unloading -= LoadContext_Unloading;
                     _loadContext = null;
                 }
+
+                if (_authMigrationAutoReenableTask != null)
+                {
+                    _authMigrationAutoReenableTaskCancellationTokenSource?.Cancel();
+                }
+
+                if (_authMigrationAutoReenableTaskCancellationTokenSource != null)
+                {
+                    _authMigrationAutoReenableTaskCancellationTokenSource?.Dispose();
+                    _authMigrationAutoReenableTaskCancellationTokenSource = null;
+                }
+
                 _httpTraceSubscription?.Dispose();
                 _diagListenerSubscription?.Dispose();
                 _traceManager?.Dispose();
@@ -629,7 +744,7 @@ namespace GitHub.Runner.Common
                     payload[0] = Enum.Parse(typeof(GitHub.Services.Common.VssCredentialsType), ((int)payload[0]).ToString());
                 }
 
-                if (payload.Length > 0)
+                if (payload.Length > 0 && !string.IsNullOrEmpty(eventData.Message))
                 {
                     message = String.Format(eventData.Message.Replace("%n", Environment.NewLine), payload);
                 }

src/Runner.Common/RunServer.cs

@@ -18,7 +18,7 @@ namespace GitHub.Runner.Common
     {
         Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
 
-        Task<AgentJobRequestMessage> GetJobMessageAsync(string id, CancellationToken token);
+        Task<AgentJobRequestMessage> GetJobMessageAsync(string id, string billingOwnerId, CancellationToken token);
 
         Task CompleteJobAsync(
             Guid planId,
@@ -28,6 +28,8 @@ namespace GitHub.Runner.Common
             IList<StepResult> stepResults,
             IList<Annotation> jobAnnotations,
             string environmentUrl,
+            IList<Telemetry> telemetry,
+            string billingOwnerId,
             CancellationToken token);
 
         Task<RenewJobResponse> RenewJobAsync(Guid planId, Guid jobId, CancellationToken token);
@@ -57,11 +59,11 @@ namespace GitHub.Runner.Common
             }
         }
 
-        public Task<AgentJobRequestMessage> GetJobMessageAsync(string id, CancellationToken cancellationToken)
+        public Task<AgentJobRequestMessage> GetJobMessageAsync(string id, string billingOwnerId, CancellationToken cancellationToken)
         {
             CheckConnection();
             return RetryRequest<AgentJobRequestMessage>(
-                async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, VarUtil.OS, cancellationToken), cancellationToken,
+                async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, VarUtil.OS, billingOwnerId, cancellationToken), cancellationToken,
                 shouldRetry: ex =>
                     ex is not TaskOrchestrationJobNotFoundException &&          // HTTP status 404
                     ex is not TaskOrchestrationJobAlreadyAcquiredException &&   // HTTP status 409
@@ -76,11 +78,16 @@ namespace GitHub.Runner.Common
             IList<StepResult> stepResults,
             IList<Annotation> jobAnnotations,
             string environmentUrl,
+            IList<Telemetry> telemetry,
+            string billingOwnerId,
             CancellationToken cancellationToken)
         {
             CheckConnection();
             return RetryRequest(
-                async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, cancellationToken), cancellationToken);
+                async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, telemetry, billingOwnerId, cancellationToken), cancellationToken,
+                shouldRetry: ex =>
+                    ex is not VssUnauthorizedException &&               // HTTP status 401
+                    ex is not TaskOrchestrationJobNotFoundException);   // HTTP status 404
         }
 
         public Task<RenewJobResponse> RenewJobAsync(Guid planId, Guid jobId, CancellationToken cancellationToken)

src/Runner.Common/Runner.Common.csproj

@@ -1,11 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <OutputType>Library</OutputType>
     <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
     <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
     <Version>$(Version)</Version>
   </PropertyGroup>
 
@@ -15,11 +15,11 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
+    <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
+    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
-    <PackageReference Include="System.Text.Encoding.CodePages" Version="4.4.0" />
+    <PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
-    <PackageReference Include="System.Threading.Channels" Version="4.4.0" />
+    <PackageReference Include="System.Threading.Channels" Version="8.0.0" />
   </ItemGroup>
 
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">

src/Runner.Common/RunnerDotcomServer.cs

@@ -46,7 +46,11 @@ namespace GitHub.Runner.Common
             var githubApiUrl = "";
             var gitHubUrlBuilder = new UriBuilder(githubUrl);
             var path = gitHubUrlBuilder.Path.Split('/', '\\', StringSplitOptions.RemoveEmptyEntries);
-            if (path.Length == 1)
+            var isOrgRunner = path.Length == 1;
+            var isRepoOrEnterpriseRunner = path.Length == 2;
+            var isRepoRunner = isRepoOrEnterpriseRunner && !string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase);
+
+            if (isOrgRunner)
             {
                 // org runner
                 if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
@@ -58,21 +62,31 @@ namespace GitHub.Runner.Common
                     githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/orgs/{path[0]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
                 }
             }
-            else if (path.Length == 2)
+            else if (isRepoOrEnterpriseRunner)
             {
-                // repo or enterprise runner.
+                // Repository runner
-                if (!string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase))
+                if (isRepoRunner)
                 {
-                    return null;
+                    if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
-                }
+                    {
-
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/repos/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
-                if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
+                    }
-                {
+                    else
-                    githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/repos/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
+                    }
                 }
                 else
                 {
-                    githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
+                    // Enterprise runner
+                    if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
+                    }
+                    else
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
+                    }
                 }
             }
             else
@@ -90,7 +104,11 @@ namespace GitHub.Runner.Common
             var githubApiUrl = "";
             var gitHubUrlBuilder = new UriBuilder(githubUrl);
             var path = gitHubUrlBuilder.Path.Split('/', '\\', StringSplitOptions.RemoveEmptyEntries);
-            if (path.Length == 1)
+            var isOrgRunner = path.Length == 1;
+            var isRepoOrEnterpriseRunner = path.Length == 2;
+            var isRepoRunner = isRepoOrEnterpriseRunner && !string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase);
+
+            if (isOrgRunner)
             {
                 // org runner
                 if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
@@ -102,21 +120,31 @@ namespace GitHub.Runner.Common
                     githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/orgs/{path[0]}/actions/runner-groups";
                 }
             }
-            else if (path.Length == 2)
+            else if (isRepoOrEnterpriseRunner)
             {
-                // repo or enterprise runner.
+                // Repository Runner
-                if (!string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase))
+                if (isRepoRunner)
                 {
-                    return null;
+                    if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
-                }
+                    {
-
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/repos/{path[0]}/{path[1]}/actions/runner-groups";
-                if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
+                    }
-                {
+                    else
-                    githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runner-groups";
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/repos/{path[0]}/{path[1]}/actions/runner-groups";
+                    }
                 }
                 else
                 {
-                    githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runner-groups";
+                    // Enterprise Runner
+                    if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runner-groups";
+                    }
+                    else
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runner-groups";
+                    }
                 }
             }
             else

src/Runner.Common/RunnerServer.cs

@@ -1,11 +1,11 @@
-using GitHub.DistributedTask.WebApi;
+using System;
-using System;
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
-using GitHub.Services.WebApi;
+using GitHub.DistributedTask.WebApi;
-using GitHub.Services.Common;
 using GitHub.Runner.Sdk;
+using GitHub.Services.Common;
+using GitHub.Services.WebApi;
 
 namespace GitHub.Runner.Common
 {
@@ -50,7 +50,10 @@ namespace GitHub.Runner.Common
         Task<PackageMetadata> GetPackageAsync(string packageType, string platform, string version, bool includeToken, CancellationToken cancellationToken);
 
         // agent update
-        Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, ulong agentId, string currentState, string trace);
+        Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, ulong agentId, string currentState, string trace, CancellationToken cancellationToken = default);
+
+        // runner config refresh
+        Task<string> RefreshRunnerConfigAsync(int agentId, string configType, string encodedRunnerConfig, CancellationToken cancellationToken);
     }
 
     public sealed class RunnerServer : RunnerService, IRunnerServer
@@ -315,10 +318,17 @@ namespace GitHub.Runner.Common
             return _genericTaskAgentClient.GetPackageAsync(packageType, platform, version, includeToken, cancellationToken: cancellationToken);
         }
 
-        public Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, ulong agentId, string currentState, string trace)
+        public Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, ulong agentId, string currentState, string trace, CancellationToken cancellationToken = default)
         {
             CheckConnection(RunnerConnectionType.Generic);
-            return _genericTaskAgentClient.UpdateAgentUpdateStateAsync(agentPoolId, agentId, currentState, trace);
+            return _genericTaskAgentClient.UpdateAgentUpdateStateAsync(agentPoolId, agentId, currentState, trace, cancellationToken: cancellationToken);
+        }
+
+        // runner config refresh
+        public Task<string> RefreshRunnerConfigAsync(int agentId, string configType, string encodedRunnerConfig, CancellationToken cancellationToken)
+        {
+            CheckConnection(RunnerConnectionType.Generic);
+            return _genericTaskAgentClient.RefreshRunnerConfigAsync(agentId, configType, encodedRunnerConfig, cancellationToken: cancellationToken);
         }
     }
 }

src/Runner.Common/RunnerService.cs

@@ -70,7 +70,8 @@ namespace GitHub.Runner.Common
 
         protected async Task RetryRequest(Func<Task> func,
             CancellationToken cancellationToken,
-            int maxRetryAttemptsCount = 5
+            int maxRetryAttemptsCount = 5,
+            Func<Exception, bool> shouldRetry = null
         )
         {
             async Task<Unit> wrappedFunc()
@@ -78,7 +79,7 @@ namespace GitHub.Runner.Common
                 await func();
                 return Unit.Value;
             }
-            await RetryRequest<Unit>(wrappedFunc, cancellationToken, maxRetryAttemptsCount);
+            await RetryRequest<Unit>(wrappedFunc, cancellationToken, maxRetryAttemptsCount, shouldRetry);
         }
 
         protected async Task<T> RetryRequest<T>(Func<Task<T>> func,

src/Runner.Common/Util/NodeUtil.cs

@@ -5,8 +5,8 @@ namespace GitHub.Runner.Common.Util
 {
     public static class NodeUtil
     {
-        private const string _defaultNodeVersion = "node16";
+        private const string _defaultNodeVersion = "node20";
-        public static readonly ReadOnlyCollection<string> BuiltInNodeVersions = new(new[] { "node16", "node20" });
+        public static readonly ReadOnlyCollection<string> BuiltInNodeVersions = new(new[] { "node20" });
         public static string GetInternalNodeVersion()
         {
             var forcedInternalNodeVersion = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ForcedInternalNodeVersion);

src/Runner.Listener/BrokerMessageListener.cs

@@ -9,11 +9,12 @@ using System.Threading;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
+using GitHub.Runner.Common.Util;
 using GitHub.Runner.Listener.Configuration;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
-using GitHub.Runner.Common.Util;
 using GitHub.Services.OAuth;
+using GitHub.Services.WebApi;
 
 namespace GitHub.Runner.Listener
 {
@@ -25,21 +26,27 @@ namespace GitHub.Runner.Listener
         private TaskAgentStatus runnerStatus = TaskAgentStatus.Online;
         private CancellationTokenSource _getMessagesTokenSource;
         private VssCredentials _creds;
+        private VssCredentials _credsV2;
         private TaskAgentSession _session;
+        private IRunnerServer _runnerServer;
         private IBrokerServer _brokerServer;
+        private ICredentialManager _credMgr;
         private readonly Dictionary<string, int> _sessionCreationExceptionTracker = new();
         private bool _accessTokenRevoked = false;
         private readonly TimeSpan _sessionCreationRetryInterval = TimeSpan.FromSeconds(30);
         private readonly TimeSpan _sessionConflictRetryLimit = TimeSpan.FromMinutes(4);
         private readonly TimeSpan _clockSkewRetryLimit = TimeSpan.FromMinutes(30);
-
+        private bool _needRefreshCredsV2 = false;
+        private bool _handlerInitialized = false;
 
         public override void Initialize(IHostContext hostContext)
         {
             base.Initialize(hostContext);
 
             _term = HostContext.GetService<ITerminal>();
+            _runnerServer = HostContext.GetService<IRunnerServer>();
             _brokerServer = HostContext.GetService<IBrokerServer>();
+            _credMgr = HostContext.GetService<ICredentialManager>();
         }
 
         public async Task<CreateSessionResult> CreateSessionAsync(CancellationToken token)
@@ -49,7 +56,8 @@ namespace GitHub.Runner.Listener
             // Settings
             var configManager = HostContext.GetService<IConfigurationManager>();
             _settings = configManager.LoadSettings();
-            var serverUrl = _settings.ServerUrlV2;
+            var serverUrlV2 = _settings.ServerUrlV2;
+            var serverUrl = _settings.ServerUrl;
             Trace.Info(_settings);
 
             if (string.IsNullOrEmpty(_settings.ServerUrlV2))
@@ -59,8 +67,7 @@ namespace GitHub.Runner.Listener
 
             // Create connection.
             Trace.Info("Loading Credentials");
-            var credMgr = HostContext.GetService<ICredentialManager>();
+            _creds = _credMgr.LoadCredentials(allowAuthUrlV2: false);
-            _creds = credMgr.LoadCredentials();
 
             var agent = new TaskAgentReference
             {
@@ -83,9 +90,18 @@ namespace GitHub.Runner.Listener
                 try
                 {
                     Trace.Info("Connecting to the Broker Server...");
-                    await _brokerServer.ConnectAsync(new Uri(serverUrl), _creds);
+                    _credsV2 = _credMgr.LoadCredentials(allowAuthUrlV2: true);
+                    await _brokerServer.ConnectAsync(new Uri(serverUrlV2), _credsV2);
                     Trace.Info("VssConnection created");
 
+                    if (!string.IsNullOrEmpty(serverUrl) &&
+                        !string.Equals(serverUrl, serverUrlV2, StringComparison.OrdinalIgnoreCase))
+                    {
+                        Trace.Info("Connecting to the Runner server...");
+                        await _runnerServer.ConnectAsync(new Uri(serverUrl), _creds);
+                        Trace.Info("VssConnection created");
+                    }
+
                     _term.WriteLine();
                     _term.WriteSuccessMessage("Connected to GitHub");
                     _term.WriteLine();
@@ -100,6 +116,13 @@ namespace GitHub.Runner.Listener
                         encounteringError = false;
                     }
 
+                    if (!_handlerInitialized)
+                    {
+                        // Register event handler for auth migration state change
+                        HostContext.AuthMigrationChanged += HandleAuthMigrationChanged;
+                        _handlerInitialized = true;
+                    }
+
                     return CreateSessionResult.Success;
                 }
                 catch (OperationCanceledException) when (token.IsCancellationRequested)
@@ -118,7 +141,7 @@ namespace GitHub.Runner.Listener
                     Trace.Error("Catch exception during create session.");
                     Trace.Error(ex);
 
-                    if (ex is VssOAuthTokenRequestException vssOAuthEx && _creds.Federated is VssOAuthCredential vssOAuthCred)
+                    if (ex is VssOAuthTokenRequestException vssOAuthEx && _credsV2.Federated is VssOAuthCredential vssOAuthCred)
                     {
                         // "invalid_client" means the runner registration has been deleted from the server.
                         if (string.Equals(vssOAuthEx.Error, "invalid_client", StringComparison.OrdinalIgnoreCase))
@@ -130,7 +153,7 @@ namespace GitHub.Runner.Listener
                         // Check whether we get 401 because the runner registration already removed by the service.
                         // If the runner registration get deleted, we can't exchange oauth token.
                         Trace.Error("Test oauth app registration.");
-                        var oauthTokenProvider = new VssOAuthTokenProvider(vssOAuthCred, new Uri(serverUrl));
+                        var oauthTokenProvider = new VssOAuthTokenProvider(vssOAuthCred, new Uri(serverUrlV2));
                         var authError = await oauthTokenProvider.ValidateCredentialAsync(token);
                         if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase))
                         {
@@ -149,6 +172,12 @@ namespace GitHub.Runner.Listener
                         return CreateSessionResult.Failure;
                     }
 
+                    if (HostContext.AllowAuthMigration)
+                    {
+                        Trace.Info("Disable migration mode for 60 minutes.");
+                        HostContext.DeferAuthMigration(TimeSpan.FromMinutes(60), $"Session creation failed with exception: {ex}");
+                    }
+
                     if (!encounteringError) //print the message only on the first error
                     {
                         _term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
@@ -165,6 +194,11 @@ namespace GitHub.Runner.Listener
         {
             if (_session != null && _session.SessionId != Guid.Empty)
             {
+                if (_handlerInitialized)
+                {
+                    HostContext.AuthMigrationChanged -= HandleAuthMigrationChanged;
+                }
+
                 if (!_accessTokenRevoked)
                 {
                     using (var ts = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
@@ -207,6 +241,13 @@ namespace GitHub.Runner.Listener
                 _getMessagesTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
                 try
                 {
+                    if (_needRefreshCredsV2)
+                    {
+                        Trace.Info("Refreshing broker connection.");
+                        await RefreshBrokerConnectionAsync();
+                        _needRefreshCredsV2 = false;
+                    }
+
                     message = await _brokerServer.GetRunnerMessageAsync(_session.SessionId,
                                                                         runnerStatus,
                                                                         BuildConstants.RunnerPackage.Version,
@@ -237,10 +278,19 @@ namespace GitHub.Runner.Listener
                     Trace.Info("Runner OAuth token has been revoked. Unable to pull message.");
                     throw;
                 }
+                catch (HostedRunnerDeprovisionedException)
+                {
+                    Trace.Info("Hosted runner has been deprovisioned.");
+                    throw;
+                }
                 catch (AccessDeniedException e) when (e.ErrorCode == 1)
                 {
                     throw;
                 }
+                catch (RunnerNotFoundException)
+                {
+                    throw;
+                }
                 catch (Exception ex)
                 {
                     Trace.Error("Catch exception during get next message.");
@@ -277,6 +327,12 @@ namespace GitHub.Runner.Listener
                             encounteringError = true;
                         }
 
+                        if (HostContext.AllowAuthMigration)
+                        {
+                            Trace.Info("Disable migration mode for 60 minutes.");
+                            HostContext.DeferAuthMigration(TimeSpan.FromMinutes(60), $"Get next message failed with exception: {ex}");
+                        }
+
                         // re-create VssConnection before next retry
                         await RefreshBrokerConnectionAsync();
 
@@ -308,7 +364,7 @@ namespace GitHub.Runner.Listener
             }
         }
 
-        public async Task RefreshListenerTokenAsync(CancellationToken cancellationToken)
+        public async Task RefreshListenerTokenAsync()
         {
             await RefreshBrokerConnectionAsync();
         }
@@ -324,6 +380,7 @@ namespace GitHub.Runner.Listener
                 ex is TaskAgentPoolNotFoundException ||
                 ex is TaskAgentSessionExpiredException ||
                 ex is AccessDeniedException ||
+                ex is RunnerNotFoundException ||
                 ex is VssUnauthorizedException)
             {
                 Trace.Info($"Non-retriable exception: {ex.Message}");
@@ -410,17 +467,16 @@ namespace GitHub.Runner.Listener
 
         private async Task RefreshBrokerConnectionAsync()
         {
-            var configManager = HostContext.GetService<IConfigurationManager>();
+            Trace.Info("Reload credentials.");
-            _settings = configManager.LoadSettings();
+            _credsV2 = _credMgr.LoadCredentials(allowAuthUrlV2: true);
+            await _brokerServer.ConnectAsync(new Uri(_settings.ServerUrlV2), _credsV2);
+            Trace.Info("Connection to Broker Server recreated.");
+        }
 
-            if (string.IsNullOrEmpty(_settings.ServerUrlV2))
+        private void HandleAuthMigrationChanged(object sender, EventArgs e)
-            {
+        {
-                throw new InvalidOperationException("ServerUrlV2 is not set");
+            Trace.Info($"Auth migration changed. Current allow auth migration state: {HostContext.AllowAuthMigration}");
-            }
+            _needRefreshCredsV2 = true;
-
-            var credMgr = HostContext.GetService<ICredentialManager>();
-            VssCredentials creds = credMgr.LoadCredentials();
-            await _brokerServer.ConnectAsync(new Uri(_settings.ServerUrlV2), creds);
         }
     }
 }

src/Runner.Listener/Configuration/ConfigurationManager.cs

@@ -127,7 +127,7 @@ namespace GitHub.Runner.Listener.Configuration
                     runnerSettings.ServerUrl = inputUrl;
                     // Get the credentials
                     credProvider = GetCredentialProvider(command, runnerSettings.ServerUrl);
-                    creds = credProvider.GetVssCredentials(HostContext);
+                    creds = credProvider.GetVssCredentials(HostContext, allowAuthUrlV2: false);
                     Trace.Info("legacy vss cred retrieved");
                 }
                 else
@@ -366,7 +366,7 @@ namespace GitHub.Runner.Listener.Configuration
                     {
                         { "clientId", agent.Authorization.ClientId.ToString("D") },
                         { "authorizationUrl", agent.Authorization.AuthorizationUrl.AbsoluteUri },
-                        { "requireFipsCryptography", agent.Properties.GetValue("RequireFipsCryptography", false).ToString() }
+                        { "requireFipsCryptography", agent.Properties.GetValue("RequireFipsCryptography", true).ToString() }
                     },
                 };
 
@@ -384,7 +384,7 @@ namespace GitHub.Runner.Listener.Configuration
             if (!runnerSettings.UseV2Flow)
             {
                 var credMgr = HostContext.GetService<ICredentialManager>();
-                VssCredentials credential = credMgr.LoadCredentials();
+                VssCredentials credential = credMgr.LoadCredentials(allowAuthUrlV2: false);
                 try
                 {
                     await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), credential);
@@ -404,6 +404,20 @@ namespace GitHub.Runner.Listener.Configuration
                 }
             }
 
+            // allow the server to override the serverUrlV2 and useV2Flow
+            if (agent.Properties.TryGetValue("ServerUrlV2", out string serverUrlV2) &&
+                !string.IsNullOrEmpty(serverUrlV2))
+            {
+                Trace.Info($"Service enforced serverUrlV2: {serverUrlV2}");
+                runnerSettings.ServerUrlV2 = serverUrlV2;
+            }
+
+            if (agent.Properties.TryGetValue("UseV2Flow", out bool useV2Flow) && useV2Flow)
+            {
+                Trace.Info($"Service enforced useV2Flow: {useV2Flow}");
+                runnerSettings.UseV2Flow = useV2Flow;
+            }
+
             _term.WriteSection("Runner settings");
 
             // We will Combine() what's stored with root.  Defaults to string a relative path
@@ -505,7 +519,7 @@ namespace GitHub.Runner.Listener.Configuration
                     if (string.IsNullOrEmpty(settings.GitHubUrl))
                     {
                         var credProvider = GetCredentialProvider(command, settings.ServerUrl);
-                        creds = credProvider.GetVssCredentials(HostContext);
+                        creds = credProvider.GetVssCredentials(HostContext, allowAuthUrlV2: false);
                         Trace.Info("legacy vss cred retrieved");
                     }
                     else

src/Runner.Listener/Configuration/CredentialManager.cs

@@ -13,7 +13,7 @@ namespace GitHub.Runner.Listener.Configuration
     public interface ICredentialManager : IRunnerService
     {
         ICredentialProvider GetCredentialProvider(string credType);
-        VssCredentials LoadCredentials();
+        VssCredentials LoadCredentials(bool allowAuthUrlV2);
     }
 
     public class CredentialManager : RunnerService, ICredentialManager
@@ -40,7 +40,7 @@ namespace GitHub.Runner.Listener.Configuration
             return creds;
         }
 
-        public VssCredentials LoadCredentials()
+        public VssCredentials LoadCredentials(bool allowAuthUrlV2)
         {
             IConfigurationStore store = HostContext.GetService<IConfigurationStore>();
 
@@ -51,21 +51,16 @@ namespace GitHub.Runner.Listener.Configuration
 
             CredentialData credData = store.GetCredentials();
             var migratedCred = store.GetMigratedCredentials();
-            if (migratedCred != null)
+            if (migratedCred != null &&
+                migratedCred.Scheme == Constants.Configuration.OAuth)
             {
                 credData = migratedCred;
-
-                // Re-write .credentials with Token URL
-                store.SaveCredential(credData);
-
-                // Delete .credentials_migrated
-                store.DeleteMigratedCredential();
             }
 
             ICredentialProvider credProv = GetCredentialProvider(credData.Scheme);
             credProv.CredentialData = credData;
 
-            VssCredentials creds = credProv.GetVssCredentials(HostContext);
+            VssCredentials creds = credProv.GetVssCredentials(HostContext, allowAuthUrlV2);
 
             return creds;
         }

src/Runner.Listener/Configuration/CredentialProvider.cs

@@ -1,7 +1,7 @@
 using System;
-using GitHub.Services.Common;
 using GitHub.Runner.Common;
 using GitHub.Runner.Sdk;
+using GitHub.Services.Common;
 using GitHub.Services.OAuth;
 
 namespace GitHub.Runner.Listener.Configuration
@@ -10,7 +10,7 @@ namespace GitHub.Runner.Listener.Configuration
     {
         Boolean RequireInteractive { get; }
         CredentialData CredentialData { get; set; }
-        VssCredentials GetVssCredentials(IHostContext context);
+        VssCredentials GetVssCredentials(IHostContext context, bool allowAuthUrlV2);
         void EnsureCredential(IHostContext context, CommandSettings command, string serverUrl);
     }
 
@@ -25,7 +25,7 @@ namespace GitHub.Runner.Listener.Configuration
         public virtual Boolean RequireInteractive => false;
         public CredentialData CredentialData { get; set; }
 
-        public abstract VssCredentials GetVssCredentials(IHostContext context);
+        public abstract VssCredentials GetVssCredentials(IHostContext context, bool allowAuthUrlV2);
         public abstract void EnsureCredential(IHostContext context, CommandSettings command, string serverUrl);
     }
 
@@ -33,7 +33,7 @@ namespace GitHub.Runner.Listener.Configuration
     {
         public OAuthAccessTokenCredential() : base(Constants.Configuration.OAuthAccessToken) { }
 
-        public override VssCredentials GetVssCredentials(IHostContext context)
+        public override VssCredentials GetVssCredentials(IHostContext context, bool allowAuthUrlV2)
         {
             ArgUtil.NotNull(context, nameof(context));
             Tracing trace = context.GetTrace(nameof(OAuthAccessTokenCredential));

src/Runner.Listener/Configuration/OAuthCredential.cs

@@ -22,10 +22,18 @@ namespace GitHub.Runner.Listener.Configuration
             // Nothing to verify here
         }
 
-        public override VssCredentials GetVssCredentials(IHostContext context)
+        public override VssCredentials GetVssCredentials(IHostContext context, bool allowAuthUrlV2)
         {
             var clientId = this.CredentialData.Data.GetValueOrDefault("clientId", null);
             var authorizationUrl = this.CredentialData.Data.GetValueOrDefault("authorizationUrl", null);
+            var authorizationUrlV2 = this.CredentialData.Data.GetValueOrDefault("authorizationUrlV2", null);
+
+            if (allowAuthUrlV2 &&
+                !string.IsNullOrEmpty(authorizationUrlV2) &&
+                context.AllowAuthMigration)
+            {
+                authorizationUrl = authorizationUrlV2;
+            }
 
             // For back compat with .credential file that doesn't has 'oauthEndpointUrl' section
             var oauthEndpointUrl = this.CredentialData.Data.GetValueOrDefault("oauthEndpointUrl", authorizationUrl);

src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs

@@ -1,4 +1,5 @@
 #if OS_WINDOWS
+#pragma warning disable CA1416
 using System.IO;
 using System.Security.Cryptography;
 using System.Text;
@@ -84,4 +85,5 @@ namespace GitHub.Runner.Listener.Configuration
         }
     }
 }
+#pragma warning restore CA1416
 #endif

src/Runner.Listener/JobDispatcher.cs

@@ -545,28 +545,36 @@ namespace GitHub.Runner.Listener
                                     detailInfo = string.Join(Environment.NewLine, workerOutput);
                                     Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
 
-                                    var jobServer = await InitializeJobServerAsync(systemConnection);
+                                    try
-                                    var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
-                                    unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
-                                    switch (jobServer)
                                     {
-                                        case IJobServer js:
+                                        var jobServer = await InitializeJobServerAsync(systemConnection);
-                                            {
+                                        var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
-                                                await LogWorkerProcessUnhandledException(js, message, unhandledExceptionIssue);
+                                        unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
-                                                // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
+                                        switch (jobServer)
-                                                if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase))
+                                        {
+                                            case IJobServer js:
                                                 {
-                                                    Trace.Info($"Finish job with result 'Failed' due to IOException.");
+                                                    await LogWorkerProcessUnhandledException(js, message, unhandledExceptionIssue);
-                                                    await ForceFailJob(js, message);
+                                                    // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
-                                                }
+                                                    if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase))
+                                                    {
+                                                        Trace.Info($"Finish job with result 'Failed' due to IOException.");
+                                                        await ForceFailJob(js, message);
+                                                    }
 
+                                                    break;
+                                                }
+                                            case IRunServer rs:
+                                                await ForceFailJob(rs, message, unhandledExceptionIssue);
                                                 break;
-                                            }
+                                            default:
-                                        case IRunServer rs:
+                                                throw new NotSupportedException($"JobServer type '{jobServer.GetType().Name}' is not supported.");
-                                            await ForceFailJob(rs, message, unhandledExceptionIssue);
+                                        }
-                                            break;
+                                    }
-                                        default:
+                                    catch (Exception ex)
-                                            throw new NotSupportedException($"JobServer type '{jobServer.GetType().Name}' is not supported.");
+                                    {
+                                        Trace.Error($"Catch exception during log worker process unhandled exception.");
+                                        Trace.Error(ex);
                                     }
                                 }
 
@@ -1198,7 +1206,7 @@ namespace GitHub.Runner.Listener
                     jobAnnotations.Add(annotation.Value);
                 }
 
-                await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, CancellationToken.None);
+                await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, telemetry: null, billingOwnerId: message.BillingOwnerId, CancellationToken.None);
             }
             catch (Exception ex)
             {

src/Runner.Listener/MessageListener.cs

@@ -33,7 +33,7 @@ namespace GitHub.Runner.Listener
         Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token);
         Task DeleteMessageAsync(TaskAgentMessage message);
 
-        Task RefreshListenerTokenAsync(CancellationToken token);
+        Task RefreshListenerTokenAsync();
         void OnJobStatus(object sender, JobStatusEventArgs e);
     }
 
@@ -44,6 +44,7 @@ namespace GitHub.Runner.Listener
         private ITerminal _term;
         private IRunnerServer _runnerServer;
         private IBrokerServer _brokerServer;
+        private ICredentialManager _credMgr;
         private TaskAgentSession _session;
         private TimeSpan _getNextMessageRetryInterval;
         private bool _accessTokenRevoked = false;
@@ -54,8 +55,9 @@ namespace GitHub.Runner.Listener
         private TaskAgentStatus runnerStatus = TaskAgentStatus.Online;
         private CancellationTokenSource _getMessagesTokenSource;
         private VssCredentials _creds;
-
+        private VssCredentials _credsV2;
-        private bool _isBrokerSession = false;
+        private bool _needRefreshCredsV2 = false;
+        private bool _handlerInitialized = false;
 
         public override void Initialize(IHostContext hostContext)
         {
@@ -64,6 +66,7 @@ namespace GitHub.Runner.Listener
             _term = HostContext.GetService<ITerminal>();
             _runnerServer = HostContext.GetService<IRunnerServer>();
             _brokerServer = hostContext.GetService<IBrokerServer>();
+            _credMgr = hostContext.GetService<ICredentialManager>();
         }
 
         public async Task<CreateSessionResult> CreateSessionAsync(CancellationToken token)
@@ -78,8 +81,7 @@ namespace GitHub.Runner.Listener
 
             // Create connection.
             Trace.Info("Loading Credentials");
-            var credMgr = HostContext.GetService<ICredentialManager>();
+            _creds = _credMgr.LoadCredentials(allowAuthUrlV2: false);
-            _creds = credMgr.LoadCredentials();
 
             var agent = new TaskAgentReference
             {
@@ -113,16 +115,6 @@ namespace GitHub.Runner.Listener
                                                         _settings.PoolId,
                                                         taskAgentSession,
                                                         token);
-
-                    if (_session.BrokerMigrationMessage != null)
-                    {
-                        Trace.Info("Runner session is in migration mode: Creating Broker session with BrokerBaseUrl: {0}", _session.BrokerMigrationMessage.BrokerBaseUrl);
-
-                        await _brokerServer.UpdateConnectionIfNeeded(_session.BrokerMigrationMessage.BrokerBaseUrl, _creds);
-                        _session = await _brokerServer.CreateSessionAsync(taskAgentSession, token);
-                        _isBrokerSession = true;
-                    }
-
                     Trace.Info($"Session created.");
                     if (encounteringError)
                     {
@@ -131,6 +123,13 @@ namespace GitHub.Runner.Listener
                         encounteringError = false;
                     }
 
+                    if (!_handlerInitialized)
+                    {
+                        Trace.Info("Registering AuthMigrationChanged event handler.");
+                        HostContext.AuthMigrationChanged += HandleAuthMigrationChanged;
+                        _handlerInitialized = true;
+                    }
+
                     return CreateSessionResult.Success;
                 }
                 catch (OperationCanceledException) when (token.IsCancellationRequested)
@@ -196,16 +195,16 @@ namespace GitHub.Runner.Listener
         {
             if (_session != null && _session.SessionId != Guid.Empty)
             {
+                if (_handlerInitialized)
+                {
+                    HostContext.AuthMigrationChanged -= HandleAuthMigrationChanged;
+                }
+
                 if (!_accessTokenRevoked)
                 {
                     using (var ts = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
                     {
                         await _runnerServer.DeleteAgentSessionAsync(_settings.PoolId, _session.SessionId, ts.Token);
-
-                        if (_isBrokerSession)
-                        {
-                            await _brokerServer.DeleteSessionAsync(ts.Token);
-                        }
                     }
                 }
                 else
@@ -261,14 +260,19 @@ namespace GitHub.Runner.Listener
                     // Decrypt the message body if the session is using encryption
                     message = DecryptMessage(message);
 
-
                     if (message != null && message.MessageType == BrokerMigrationMessage.MessageType)
                     {
-                        Trace.Info("BrokerMigration message received. Polling Broker for messages...");
-
                         var migrationMessage = JsonUtility.FromString<BrokerMigrationMessage>(message.Body);
 
-                        await _brokerServer.UpdateConnectionIfNeeded(migrationMessage.BrokerBaseUrl, _creds);
+                        _credsV2 = _credMgr.LoadCredentials(allowAuthUrlV2: true);
+                        await _brokerServer.UpdateConnectionIfNeeded(migrationMessage.BrokerBaseUrl, _credsV2);
+                        if (_needRefreshCredsV2)
+                        {
+                            Trace.Info("Refreshing credentials for V2.");
+                            await _brokerServer.ForceRefreshConnection(_credsV2);
+                            _needRefreshCredsV2 = false;
+                        }
+
                         message = await _brokerServer.GetRunnerMessageAsync(_session.SessionId,
                                                                         runnerStatus,
                                                                         BuildConstants.RunnerPackage.Version,
@@ -306,15 +310,28 @@ namespace GitHub.Runner.Listener
                     _accessTokenRevoked = true;
                     throw;
                 }
+                catch (HostedRunnerDeprovisionedException)
+                {
+                    Trace.Info("Hosted runner has been deprovisioned.");
+                    throw;
+                }
                 catch (AccessDeniedException e) when (e.ErrorCode == 1)
                 {
                     throw;
                 }
+                catch (RunnerNotFoundException)
+                {
+                    throw;
+                }
                 catch (Exception ex)
                 {
                     Trace.Error("Catch exception during get next message.");
                     Trace.Error(ex);
 
+                    // clear out potential message for broker migration,
+                    // in case the exception is thrown from get message from broker-listener.
+                    message = null;
+
                     // don't retry if SkipSessionRecover = true, DT service will delete agent session to stop agent from taking more jobs.
                     if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && (await CreateSessionAsync(token) == CreateSessionResult.Success))
                     {
@@ -347,6 +364,12 @@ namespace GitHub.Runner.Listener
                             encounteringError = true;
                         }
 
+                        if (HostContext.AllowAuthMigration)
+                        {
+                            Trace.Info("Disable migration mode for 60 minutes.");
+                            HostContext.DeferAuthMigration(TimeSpan.FromMinutes(60), $"Get next message failed with exception: {ex}");
+                        }
+
                         // re-create VssConnection before next retry
                         await _runnerServer.RefreshConnectionAsync(RunnerConnectionType.MessageQueue, TimeSpan.FromSeconds(60));
 
@@ -404,10 +427,11 @@ namespace GitHub.Runner.Listener
             }
         }
 
-        public async Task RefreshListenerTokenAsync(CancellationToken cancellationToken)
+        public async Task RefreshListenerTokenAsync()
         {
             await _runnerServer.RefreshConnectionAsync(RunnerConnectionType.MessageQueue, TimeSpan.FromSeconds(60));
-            await _brokerServer.ForceRefreshConnection(_creds);
+            _credsV2 = _credMgr.LoadCredentials(allowAuthUrlV2: true);
+            await _brokerServer.ForceRefreshConnection(_credsV2);
         }
 
         private TaskAgentMessage DecryptMessage(TaskAgentMessage message)
@@ -459,6 +483,7 @@ namespace GitHub.Runner.Listener
                 ex is TaskAgentPoolNotFoundException ||
                 ex is TaskAgentSessionExpiredException ||
                 ex is AccessDeniedException ||
+                ex is RunnerNotFoundException ||
                 ex is VssUnauthorizedException)
             {
                 Trace.Info($"Non-retriable exception: {ex.Message}");
@@ -525,7 +550,8 @@ namespace GitHub.Runner.Listener
             }
             else if (ex is TaskAgentPoolNotFoundException ||
                      ex is AccessDeniedException ||
-                     ex is VssUnauthorizedException)
+                     ex is VssUnauthorizedException ||
+                     (ex is VssOAuthTokenRequestException oauthEx && oauthEx.Error != "server_error"))
             {
                 Trace.Info($"Non-retriable exception: {ex.Message}");
                 return false;
@@ -536,5 +562,11 @@ namespace GitHub.Runner.Listener
                 return true;
             }
         }
+
+        private void HandleAuthMigrationChanged(object sender, EventArgs e)
+        {
+            Trace.Info($"Auth migration changed. Current allow auth migration state: {HostContext.AllowAuthMigration}");
+            _needRefreshCredsV2 = true;
+        }
     }
 }

src/Runner.Listener/Program.cs

@@ -7,6 +7,7 @@ using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.WebApi;
+using GitHub.Services.WebApi;
 
 namespace GitHub.Runner.Listener
 {
@@ -144,6 +145,12 @@ namespace GitHub.Runner.Listener
                 trace.Error(e);
                 return Constants.Runner.ReturnCode.TerminatedError;
             }
+            catch (RunnerNotFoundException e)
+            {
+                terminal.WriteError($"An error occurred: {e.Message}");
+                trace.Error(e);
+                return Constants.Runner.ReturnCode.TerminatedError;
+            }
             catch (Exception e)
             {
                 terminal.WriteError($"An error occurred: {e.Message}");

src/Runner.Listener/Runner.Listener.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <OutputType>Exe</OutputType>
     <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
     <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
     <Version>$(Version)</Version>
     <PredefinedCulturesOnly>false</PredefinedCulturesOnly>
     <PublishReadyToRunComposite>true</PublishReadyToRunComposite>
@@ -18,11 +19,11 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
+    <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="System.IO.FileSystem.AccessControl" Version="4.4.0" />
+    <PackageReference Include="System.IO.FileSystem.AccessControl" Version="5.0.0" />
-    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
+    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
-    <PackageReference Include="System.ServiceProcess.ServiceController" Version="4.4.0" />
+    <PackageReference Include="System.ServiceProcess.ServiceController" Version="8.0.0" />
   </ItemGroup>
 
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">

src/Runner.Listener/Runner.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -31,6 +32,13 @@ namespace GitHub.Runner.Listener
         private ITerminal _term;
         private bool _inConfigStage;
         private ManualResetEvent _completedCommand = new(false);
+        private readonly ConcurrentQueue<string> _authMigrationTelemetries = new();
+        private Task _authMigrationTelemetryTask;
+        private readonly object _authMigrationTelemetryLock = new();
+        private IRunnerServer _runnerServer;
+        private CancellationTokenSource _authMigrationTelemetryTokenSource = new();
+        private bool _runnerExiting = false;
+        private bool _hasTerminationGracePeriod = false;
 
         // <summary>
         // Helps avoid excessive calls to Run Service when encountering non-retriable errors from /acquirejob.
@@ -51,6 +59,7 @@ namespace GitHub.Runner.Listener
             base.Initialize(hostContext);
             _term = HostContext.GetService<ITerminal>();
             _acquireJobThrottler = HostContext.CreateService<IErrorThrottler>();
+            _runnerServer = HostContext.GetService<IRunnerServer>();
         }
 
         public async Task<int> ExecuteCommand(CommandSettings command)
@@ -66,6 +75,8 @@ namespace GitHub.Runner.Listener
                 //register a SIGTERM handler
                 HostContext.Unloading += Runner_Unloading;
 
+                HostContext.AuthMigrationChanged += HandleAuthMigrationChanged;
+
                 // TODO Unit test to cover this logic
                 Trace.Info(nameof(ExecuteCommand));
                 var configManager = HostContext.GetService<IConfigurationManager>();
@@ -228,10 +239,12 @@ namespace GitHub.Runner.Listener
                             var configFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), config.Key);
                             var configContent = Convert.FromBase64String(config.Value);
 #if OS_WINDOWS
+#pragma warning disable CA1416
                             if (configFile == HostContext.GetConfigFile(WellKnownConfigFile.RSACredentials))
                             {
                                 configContent = ProtectedData.Protect(configContent, null, DataProtectionScope.LocalMachine);
                             }
+#pragma warning restore CA1416
 #endif
                             File.WriteAllBytes(configFile, configContent);
                             File.SetAttributes(configFile, File.GetAttributes(configFile) | FileAttributes.Hidden);
@@ -298,6 +311,21 @@ namespace GitHub.Runner.Listener
                         _term.WriteLine("https://docs.github.com/en/actions/hosting-your-own-runners/autoscaling-with-self-hosted-runners#using-ephemeral-runners-for-autoscaling", ConsoleColor.Yellow);
                     }
 
+                    if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable(Constants.Variables.Agent.ActionsTerminationGracePeriodSeconds)))
+                    {
+                        _hasTerminationGracePeriod = true;
+                        Trace.Verbose($"Runner has termination grace period set");
+                    }
+
+                    var cred = store.GetCredentials();
+                    if (cred != null &&
+                        cred.Scheme == Constants.Configuration.OAuth &&
+                        cred.Data.ContainsKey("EnableAuthMigrationByDefault"))
+                    {
+                        Trace.Info("Enable auth migration by default.");
+                        HostContext.EnableAuthMigration("EnableAuthMigrationByDefault");
+                    }
+
                     // Run the runner interactively or as service
                     return await RunAsync(settings, command.RunOnce || settings.Ephemeral);
                 }
@@ -309,6 +337,8 @@ namespace GitHub.Runner.Listener
             }
             finally
             {
+                _authMigrationTelemetryTokenSource?.Cancel();
+                HostContext.AuthMigrationChanged -= HandleAuthMigrationChanged;
                 _term.CancelKeyPress -= CtrlCHandler;
                 HostContext.Unloading -= Runner_Unloading;
                 _completedCommand.Set();
@@ -317,9 +347,10 @@ namespace GitHub.Runner.Listener
 
         private void Runner_Unloading(object sender, EventArgs e)
         {
+            _runnerExiting = true;
             if ((!_inConfigStage) && (!HostContext.RunnerShutdownToken.IsCancellationRequested))
             {
-                HostContext.ShutdownRunner(ShutdownReason.UserCancelled);
+                HostContext.ShutdownRunner(ShutdownReason.UserCancelled, GetShutdownDelay());
                 _completedCommand.WaitOne(Constants.Runner.ExitOnUnloadTimeout);
             }
         }
@@ -327,6 +358,7 @@ namespace GitHub.Runner.Listener
         private void CtrlCHandler(object sender, EventArgs e)
         {
             _term.WriteLine("Exiting...");
+            _runnerExiting = true;
             if (_inConfigStage)
             {
                 HostContext.Dispose();
@@ -349,16 +381,28 @@ namespace GitHub.Runner.Listener
                         reason = ShutdownReason.UserCancelled;
                     }
 
-                    HostContext.ShutdownRunner(reason);
+                    HostContext.ShutdownRunner(reason, GetShutdownDelay());
                 }
                 else
                 {
-                    HostContext.ShutdownRunner(ShutdownReason.UserCancelled);
+                    HostContext.ShutdownRunner(ShutdownReason.UserCancelled, GetShutdownDelay());
                 }
             }
         }
 
-        private IMessageListener GetMesageListener(RunnerSettings settings)
+        private void HandleJobStatusEvent(object sender, JobStatusEventArgs e)
+        {
+            if (_hasTerminationGracePeriod &&
+                e != null &&
+                e.Status != TaskAgentStatus.Busy &&
+                _runnerExiting)
+            {
+                Trace.Info("Runner is no longer busy, shutting down.");
+                HostContext.ShutdownRunner(ShutdownReason.UserCancelled);
+            }
+        }
+
+        private IMessageListener GetMessageListener(RunnerSettings settings)
         {
             if (settings.UseV2Flow)
             {
@@ -377,7 +421,7 @@ namespace GitHub.Runner.Listener
             try
             {
                 Trace.Info(nameof(RunAsync));
-                _listener = GetMesageListener(settings);
+                _listener = GetMessageListener(settings);
                 CreateSessionResult createSessionResult = await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken);
                 if (createSessionResult == CreateSessionResult.SessionConflict)
                 {
@@ -408,9 +452,13 @@ namespace GitHub.Runner.Listener
                     bool autoUpdateInProgress = false;
                     Task<bool> selfUpdateTask = null;
                     bool runOnceJobReceived = false;
-                    jobDispatcher = HostContext.CreateService<IJobDispatcher>();
+                    jobDispatcher = HostContext.GetService<IJobDispatcher>();
 
                     jobDispatcher.JobStatus += _listener.OnJobStatus;
+                    if (_hasTerminationGracePeriod)
+                    {
+                        jobDispatcher.JobStatus += HandleJobStatusEvent;
+                    }
 
                     while (!HostContext.RunnerShutdownToken.IsCancellationRequested)
                     {
@@ -568,21 +616,21 @@ namespace GitHub.Runner.Listener
 
                                     // Create connection
                                     var credMgr = HostContext.GetService<ICredentialManager>();
-                                    var creds = credMgr.LoadCredentials();
-
                                     if (string.IsNullOrEmpty(messageRef.RunServiceUrl))
                                     {
+                                        var creds = credMgr.LoadCredentials(allowAuthUrlV2: false);
                                         var actionsRunServer = HostContext.CreateService<IActionsRunServer>();
                                         await actionsRunServer.ConnectAsync(new Uri(settings.ServerUrl), creds);
                                         jobRequestMessage = await actionsRunServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageQueueLoopTokenSource.Token);
                                     }
                                     else
                                     {
+                                        var credsV2 = credMgr.LoadCredentials(allowAuthUrlV2: true);
                                         var runServer = HostContext.CreateService<IRunServer>();
-                                        await runServer.ConnectAsync(new Uri(messageRef.RunServiceUrl), creds);
+                                        await runServer.ConnectAsync(new Uri(messageRef.RunServiceUrl), credsV2);
                                         try
                                         {
-                                            jobRequestMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageQueueLoopTokenSource.Token);
+                                            jobRequestMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageRef.BillingOwnerId, messageQueueLoopTokenSource.Token);
                                             _acquireJobThrottler.Reset();
                                         }
                                         catch (Exception ex) when (
@@ -597,6 +645,13 @@ namespace GitHub.Runner.Listener
                                         catch (Exception ex)
                                         {
                                             Trace.Error($"Caught exception from acquiring job message: {ex}");
+
+                                            if (HostContext.AllowAuthMigration)
+                                            {
+                                                Trace.Info("Disable migration mode for 60 minutes.");
+                                                HostContext.DeferAuthMigration(TimeSpan.FromMinutes(60), $"Acquire job failed with exception: {ex}");
+                                            }
+
                                             continue;
                                         }
                                     }
@@ -631,7 +686,18 @@ namespace GitHub.Runner.Listener
                             else if (string.Equals(message.MessageType, TaskAgentMessageTypes.ForceTokenRefresh))
                             {
                                 Trace.Info("Received ForceTokenRefreshMessage");
-                                await _listener.RefreshListenerTokenAsync(messageQueueLoopTokenSource.Token);
+                                await _listener.RefreshListenerTokenAsync();
+                            }
+                            else if (string.Equals(message.MessageType, RunnerRefreshConfigMessage.MessageType))
+                            {
+                                var runnerRefreshConfigMessage = JsonUtility.FromString<RunnerRefreshConfigMessage>(message.Body);
+                                Trace.Info($"Received RunnerRefreshConfigMessage for '{runnerRefreshConfigMessage.ConfigType}' config file");
+                                var configUpdater = HostContext.GetService<IRunnerConfigUpdater>();
+                                await configUpdater.UpdateRunnerConfigAsync(
+                                    runnerQualifiedId: runnerRefreshConfigMessage.RunnerQualifiedId,
+                                    configType: runnerRefreshConfigMessage.ConfigType,
+                                    serviceType: runnerRefreshConfigMessage.ServiceType,
+                                    configRefreshUrl: runnerRefreshConfigMessage.ConfigRefreshUrl);
                             }
                             else
                             {
@@ -663,6 +729,10 @@ namespace GitHub.Runner.Listener
                 {
                     if (jobDispatcher != null)
                     {
+                        if (_hasTerminationGracePeriod)
+                        {
+                            jobDispatcher.JobStatus -= HandleJobStatusEvent;
+                        }
                         jobDispatcher.JobStatus -= _listener.OnJobStatus;
                         await jobDispatcher.ShutdownAsync();
                     }
@@ -695,10 +765,109 @@ namespace GitHub.Runner.Listener
             {
                 Trace.Info("Runner OAuth token has been revoked. Shutting down.");
             }
+            catch (HostedRunnerDeprovisionedException)
+            {
+                Trace.Info("Hosted runner has been deprovisioned. Shutting down.");
+            }
 
             return Constants.Runner.ReturnCode.Success;
         }
 
+        private void HandleAuthMigrationChanged(object sender, AuthMigrationEventArgs e)
+        {
+            Trace.Verbose("Handle AuthMigrationChanged in Runner");
+            _authMigrationTelemetries.Enqueue($"{DateTime.UtcNow.ToString("O")}: {e.Trace}");
+
+            // only start the telemetry reporting task once auth migration is changed (enabled or disabled)
+            lock (_authMigrationTelemetryLock)
+            {
+                if (_authMigrationTelemetryTask == null)
+                {
+                    _authMigrationTelemetryTask = ReportAuthMigrationTelemetryAsync(_authMigrationTelemetryTokenSource.Token);
+                }
+            }
+        }
+
+        private async Task ReportAuthMigrationTelemetryAsync(CancellationToken token)
+        {
+            var configManager = HostContext.GetService<IConfigurationManager>();
+            var runnerSettings = configManager.LoadSettings();
+
+            while (!token.IsCancellationRequested)
+            {
+                try
+                {
+                    await HostContext.Delay(TimeSpan.FromSeconds(60), token);
+                }
+                catch (TaskCanceledException)
+                {
+                    // Ignore cancellation
+                }
+
+                Trace.Verbose("Checking for auth migration telemetry to report");
+                while (_authMigrationTelemetries.TryDequeue(out var telemetry))
+                {
+                    Trace.Verbose($"Reporting auth migration telemetry: {telemetry}");
+                    if (runnerSettings != null)
+                    {
+                        try
+                        {
+                            using (var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
+                            {
+                                await _runnerServer.UpdateAgentUpdateStateAsync(runnerSettings.PoolId, runnerSettings.AgentId, "RefreshConfig", telemetry, tokenSource.Token);
+                            }
+                        }
+                        catch (Exception ex)
+                        {
+                            Trace.Error("Failed to report auth migration telemetry.");
+                            Trace.Error(ex);
+                            _authMigrationTelemetries.Enqueue(telemetry);
+                        }
+                    }
+
+                    if (!token.IsCancellationRequested)
+                    {
+                        try
+                        {
+                            await HostContext.Delay(TimeSpan.FromSeconds(10), token);
+                        }
+                        catch (TaskCanceledException)
+                        {
+                            // Ignore cancellation
+                        }
+                    }
+                }
+            }
+        }
+
+        private TimeSpan GetShutdownDelay()
+        {
+            TimeSpan delay = TimeSpan.Zero;
+            if (_hasTerminationGracePeriod)
+            {
+                var jobDispatcher = HostContext.GetService<IJobDispatcher>();
+                if (jobDispatcher.Busy)
+                {
+                    Trace.Info("Runner is busy, checking for grace period.");
+                    var delayEnv = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ActionsTerminationGracePeriodSeconds);
+                    if (!string.IsNullOrEmpty(delayEnv) &&
+                        int.TryParse(delayEnv, out int delaySeconds) &&
+                        delaySeconds > 0 &&
+                        delaySeconds < 60 * 60) // 1 hour
+                    {
+                        Trace.Info($"Waiting for {delaySeconds} seconds before shutting down.");
+                        delay = TimeSpan.FromSeconds(delaySeconds);
+                    }
+                }
+                else
+                {
+                    Trace.Verbose("Runner is not busy, no grace period.");
+                }
+            }
+
+            return delay;
+        }
+
         private void PrintUsage(CommandSettings command)
         {
             string separator;

src/Runner.Listener/RunnerConfigUpdater.cs

@@ -0,0 +1,287 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.Runner.Common;
+using GitHub.Runner.Sdk;
+using GitHub.Services.Common;
+
+namespace GitHub.Runner.Listener
+{
+    [ServiceLocator(Default = typeof(RunnerConfigUpdater))]
+    public interface IRunnerConfigUpdater : IRunnerService
+    {
+        Task UpdateRunnerConfigAsync(string runnerQualifiedId, string configType, string serviceType, string configRefreshUrl);
+    }
+
+    public sealed class RunnerConfigUpdater : RunnerService, IRunnerConfigUpdater
+    {
+        private RunnerSettings _settings;
+        private CredentialData _credData;
+        private IRunnerServer _runnerServer;
+        private IConfigurationStore _store;
+
+        public override void Initialize(IHostContext hostContext)
+        {
+            base.Initialize(hostContext);
+            _store = hostContext.GetService<IConfigurationStore>();
+            _settings = _store.GetSettings();
+            _credData = _store.GetCredentials();
+            _runnerServer = HostContext.GetService<IRunnerServer>();
+        }
+
+        public async Task UpdateRunnerConfigAsync(string runnerQualifiedId, string configType, string serviceType, string configRefreshUrl)
+        {
+            Trace.Entering();
+            try
+            {
+                ArgUtil.NotNullOrEmpty(runnerQualifiedId, nameof(runnerQualifiedId));
+                ArgUtil.NotNullOrEmpty(configType, nameof(configType));
+                ArgUtil.NotNullOrEmpty(serviceType, nameof(serviceType));
+                ArgUtil.NotNullOrEmpty(configRefreshUrl, nameof(configRefreshUrl));
+
+                // make sure the runner qualified id matches the current runner
+                if (!await VerifyRunnerQualifiedId(runnerQualifiedId))
+                {
+                    return;
+                }
+
+                // keep the timeout short to avoid blocking the main thread
+                using (var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
+                {
+                    switch (configType.ToLowerInvariant())
+                    {
+                        case "runner":
+                            await UpdateRunnerSettingsAsync(serviceType, configRefreshUrl, tokenSource.Token);
+                            break;
+                        case "credentials":
+                            await UpdateRunnerCredentialsAsync(serviceType, configRefreshUrl, tokenSource.Token);
+                            break;
+                        default:
+                            Trace.Error($"Invalid config type '{configType}'.");
+                            await ReportTelemetryAsync($"Invalid config type '{configType}'.");
+                            return;
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Trace.Error($"Failed to update runner '{configType}' config.");
+                Trace.Error(ex);
+                await ReportTelemetryAsync($"Failed to update runner '{configType}' config: {ex}");
+            }
+        }
+
+        private async Task UpdateRunnerSettingsAsync(string serviceType, string configRefreshUrl, CancellationToken token)
+        {
+            Trace.Entering();
+            // read the current runner settings and encode with base64
+            var runnerConfig = HostContext.GetConfigFile(WellKnownConfigFile.Runner);
+            string runnerConfigContent = File.ReadAllText(runnerConfig, Encoding.UTF8);
+            var encodedConfig = Convert.ToBase64String(Encoding.UTF8.GetBytes(runnerConfigContent));
+            if (string.IsNullOrEmpty(encodedConfig))
+            {
+                await ReportTelemetryAsync("Failed to get encoded runner settings.");
+                return;
+            }
+
+            // exchange the encoded runner settings with the service
+            string refreshedEncodedConfig = await RefreshRunnerConfigAsync(encodedConfig, serviceType, "runner", configRefreshUrl, token);
+            if (string.IsNullOrEmpty(refreshedEncodedConfig))
+            {
+                // service will return empty string if there is no change in the config
+                return;
+            }
+
+            var decodedConfig = Encoding.UTF8.GetString(Convert.FromBase64String(refreshedEncodedConfig));
+            RunnerSettings refreshedRunnerConfig;
+            try
+            {
+                refreshedRunnerConfig = StringUtil.ConvertFromJson<RunnerSettings>(decodedConfig);
+            }
+            catch (Exception ex)
+            {
+                Trace.Error($"Failed to convert runner config from json '{decodedConfig}'.");
+                Trace.Error(ex);
+                await ReportTelemetryAsync($"Failed to convert runner config '{decodedConfig}' from json: {ex}");
+                return;
+            }
+
+            // make sure the runner id and name in the refreshed config match the current runner
+            if (refreshedRunnerConfig?.AgentId != _settings.AgentId)
+            {
+                Trace.Error($"Runner id in refreshed config '{refreshedRunnerConfig?.AgentId.ToString() ?? "Empty"}' does not match the current runner '{_settings.AgentId}'.");
+                await ReportTelemetryAsync($"Runner id in refreshed config '{refreshedRunnerConfig?.AgentId.ToString() ?? "Empty"}' does not match the current runner '{_settings.AgentId}'.");
+                return;
+            }
+
+            if (refreshedRunnerConfig?.AgentName != _settings.AgentName)
+            {
+                Trace.Error($"Runner name in refreshed config '{refreshedRunnerConfig?.AgentName ?? "Empty"}' does not match the current runner '{_settings.AgentName}'.");
+                await ReportTelemetryAsync($"Runner name in refreshed config '{refreshedRunnerConfig?.AgentName ?? "Empty"}' does not match the current runner '{_settings.AgentName}'.");
+                return;
+            }
+
+            // save the refreshed runner settings as a separate file
+            _store.SaveMigratedSettings(refreshedRunnerConfig);
+            await ReportTelemetryAsync("Runner settings updated successfully.");
+        }
+
+        private async Task UpdateRunnerCredentialsAsync(string serviceType, string configRefreshUrl, CancellationToken token)
+        {
+            Trace.Entering();
+            // read the current runner credentials and encode with base64
+            var credConfig = HostContext.GetConfigFile(WellKnownConfigFile.Credentials);
+            string credConfigContent = File.ReadAllText(credConfig, Encoding.UTF8);
+            var encodedConfig = Convert.ToBase64String(Encoding.UTF8.GetBytes(credConfigContent));
+            if (string.IsNullOrEmpty(encodedConfig))
+            {
+                await ReportTelemetryAsync("Failed to get encoded credentials.");
+                return;
+            }
+
+            CredentialData currentCred = _store.GetCredentials();
+            if (currentCred == null)
+            {
+                await ReportTelemetryAsync("Failed to get current credentials.");
+                return;
+            }
+
+            // we only support refreshing OAuth credentials which is used by self-hosted runners.
+            if (currentCred.Scheme != Constants.Configuration.OAuth)
+            {
+                await ReportTelemetryAsync($"Not supported credential scheme '{currentCred.Scheme}'.");
+                return;
+            }
+
+            // exchange the encoded runner credentials with the service
+            string refreshedEncodedConfig = await RefreshRunnerConfigAsync(encodedConfig, serviceType, "credentials", configRefreshUrl, token);
+            if (string.IsNullOrEmpty(refreshedEncodedConfig))
+            {
+                // service will return empty string if there is no change in the config
+                return;
+            }
+
+            var decodedConfig = Encoding.UTF8.GetString(Convert.FromBase64String(refreshedEncodedConfig));
+            CredentialData refreshedCredConfig;
+            try
+            {
+                refreshedCredConfig = StringUtil.ConvertFromJson<CredentialData>(decodedConfig);
+            }
+            catch (Exception ex)
+            {
+                Trace.Error($"Failed to convert credentials config from json '{decodedConfig}'.");
+                Trace.Error(ex);
+                await ReportTelemetryAsync($"Failed to convert credentials config '{decodedConfig}' from json: {ex}");
+                return;
+            }
+
+            // make sure the credential scheme in the refreshed config match the current credential scheme
+            if (refreshedCredConfig?.Scheme != _credData.Scheme)
+            {
+                Trace.Error($"Credential scheme in refreshed config '{refreshedCredConfig?.Scheme ?? "Empty"}' does not match the current credential scheme '{_credData.Scheme}'.");
+                await ReportTelemetryAsync($"Credential scheme in refreshed config '{refreshedCredConfig?.Scheme ?? "Empty"}' does not match the current credential scheme '{_credData.Scheme}'.");
+                return;
+            }
+
+            if (_credData.Scheme == Constants.Configuration.OAuth)
+            {
+                // make sure the credential clientId in the refreshed config match the current credential clientId for OAuth auth scheme
+                var clientId = _credData.Data.GetValueOrDefault("clientId", null);
+                var refreshedClientId = refreshedCredConfig.Data.GetValueOrDefault("clientId", null);
+                if (clientId != refreshedClientId)
+                {
+                    Trace.Error($"Credential clientId in refreshed config '{refreshedClientId ?? "Empty"}' does not match the current credential clientId '{clientId}'.");
+                    await ReportTelemetryAsync($"Credential clientId in refreshed config '{refreshedClientId ?? "Empty"}' does not match the current credential clientId '{clientId}'.");
+                    return;
+                }
+
+                //  make sure the credential authorizationUrl in the refreshed config match the current credential authorizationUrl for OAuth auth scheme
+                var authorizationUrl = _credData.Data.GetValueOrDefault("authorizationUrl", null);
+                var refreshedAuthorizationUrl = refreshedCredConfig.Data.GetValueOrDefault("authorizationUrl", null);
+                if (authorizationUrl != refreshedAuthorizationUrl)
+                {
+                    Trace.Error($"Credential authorizationUrl in refreshed config '{refreshedAuthorizationUrl ?? "Empty"}' does not match the current credential authorizationUrl '{authorizationUrl}'.");
+                    await ReportTelemetryAsync($"Credential authorizationUrl in refreshed config '{refreshedAuthorizationUrl ?? "Empty"}' does not match the current credential authorizationUrl '{authorizationUrl}'.");
+                    return;
+                }
+            }
+
+            // save the refreshed runner credentials as a separate file
+            _store.SaveMigratedCredential(refreshedCredConfig);
+
+            if (refreshedCredConfig.Data.ContainsKey("authorizationUrlV2"))
+            {
+                HostContext.EnableAuthMigration("Credential file updated");
+                await ReportTelemetryAsync("Runner credentials updated successfully. Auth migration is enabled.");
+            }
+            else
+            {
+                HostContext.DeferAuthMigration(TimeSpan.FromDays(365), "Credential file does not contain authorizationUrlV2");
+                await ReportTelemetryAsync("Runner credentials updated successfully. Auth migration is disabled.");
+            }
+        }
+
+        private async Task<bool> VerifyRunnerQualifiedId(string runnerQualifiedId)
+        {
+            Trace.Entering();
+            Trace.Info($"Verifying runner qualified id: {runnerQualifiedId}");
+            var idParts = runnerQualifiedId.Split("/", StringSplitOptions.RemoveEmptyEntries);
+            if (idParts.Length != 4 || idParts[3] != _settings.AgentId.ToString())
+            {
+                Trace.Error($"Runner qualified id '{runnerQualifiedId}' does not match the current runner '{_settings.AgentId}'.");
+                await ReportTelemetryAsync($"Runner qualified id '{runnerQualifiedId}' does not match the current runner '{_settings.AgentId}'.");
+                return false;
+            }
+            return true;
+        }
+
+        private async Task<string> RefreshRunnerConfigAsync(string encodedConfig, string serviceType, string configType, string configRefreshUrl, CancellationToken token)
+        {
+            string refreshedEncodedConfig;
+            switch (serviceType.ToLowerInvariant())
+            {
+                case "pipelines":
+                    try
+                    {
+                        refreshedEncodedConfig = await _runnerServer.RefreshRunnerConfigAsync((int)_settings.AgentId, configType, encodedConfig, token);
+                    }
+                    catch (Exception ex)
+                    {
+                        Trace.Error($"Failed to refresh runner {configType} config with service.");
+                        Trace.Error(ex);
+                        await ReportTelemetryAsync($"Failed to refresh {configType} config: {ex}");
+                        return null;
+                    }
+                    break;
+                case "runner-admin":
+                    throw new NotSupportedException("Runner admin service is not supported.");
+                default:
+                    Trace.Error($"Invalid service type '{serviceType}'.");
+                    await ReportTelemetryAsync($"Invalid service type '{serviceType}'.");
+                    return null;
+            }
+
+            return refreshedEncodedConfig;
+        }
+
+        private async Task ReportTelemetryAsync(string telemetry)
+        {
+            Trace.Entering();
+            try
+            {
+                using (var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
+                {
+                    await _runnerServer.UpdateAgentUpdateStateAsync(_settings.PoolId, _settings.AgentId, "RefreshConfig", telemetry, tokenSource.Token);
+                }
+            }
+            catch (Exception ex)
+            {
+                Trace.Error("Failed to report telemetry.");
+                Trace.Error(ex);
+            }
+        }
+    }
+}

src/Runner.Listener/RunnerJobRequestRef.cs

@@ -7,9 +7,14 @@ namespace GitHub.Runner.Listener
     {
         [DataMember(Name = "id")]
         public string Id { get; set; }
+        
         [DataMember(Name = "runner_request_id")]
         public string RunnerRequestId { get; set; }
+        
         [DataMember(Name = "run_service_url")]
         public string RunServiceUrl { get; set; }
+        
+        [DataMember(Name = "billing_owner_id")]
+        public string BillingOwnerId { get; set; }
     }
 }

src/Runner.PluginHost/Runner.PluginHost.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <OutputType>Exe</OutputType>
     <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
     <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
     <Version>$(Version)</Version>
     <PredefinedCulturesOnly>false</PredefinedCulturesOnly>
     <PublishReadyToRunComposite>true</PublishReadyToRunComposite>

src/Runner.Plugins/Runner.Plugins.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <OutputType>Library</OutputType>
     <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
     <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
     <Version>$(Version)</Version>
   </PropertyGroup>
 

src/Runner.Sdk/Runner.Sdk.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <OutputType>Library</OutputType>
     <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
     <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
     <Version>$(Version)</Version>
   </PropertyGroup>
 
@@ -14,9 +15,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="System.Text.Encoding.CodePages" Version="4.4.0" />
+    <PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
+    <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
-    <PackageReference Include="System.Threading.Channels" Version="4.4.0" />
+    <PackageReference Include="System.Threading.Channels" Version="8.0.0" />
   </ItemGroup>
 
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">

src/Runner.Sdk/Util/UrlUtil.cs

@@ -60,5 +60,15 @@ namespace GitHub.Runner.Sdk
             }
             return string.Empty;
         }
+
+        public static string GetVssRequestId(HttpResponseHeaders headers)
+        {
+            if (headers != null &&
+                headers.TryGetValues("x-vss-e2eid", out var headerValues))
+            {
+                return headerValues.FirstOrDefault();
+            }
+            return string.Empty;
+        }
     }
 }

src/Runner.Worker/ActionManager.cs

@@ -775,7 +775,19 @@ namespace GitHub.Runner.Worker
                 // make sure we get a clean folder ready to use.
                 IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
                 Directory.CreateDirectory(destDirectory);
-                executionContext.Output($"Download action repository '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}' (SHA:{downloadInfo.ResolvedSha})");
+
+                if (downloadInfo.PackageDetails != null) 
+                {
+                    executionContext.Output($"##[group]Download immutable action package '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}'");
+                    executionContext.Output($"Version: {downloadInfo.PackageDetails.Version}");
+                    executionContext.Output($"Digest: {downloadInfo.PackageDetails.ManifestDigest}");
+                    executionContext.Output($"Source commit SHA: {downloadInfo.ResolvedSha}");
+                    executionContext.Output("##[endgroup]");
+                } 
+                else 
+                {
+                    executionContext.Output($"Download action repository '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}' (SHA:{downloadInfo.ResolvedSha})");
+                }
             }
 
             //download and extract action in a temp folder and rename it on success

src/Runner.Worker/ExecutionContext.cs

@@ -508,6 +508,9 @@ namespace GitHub.Runner.Worker
                     Status = _record.State,
                     Number = _record.Order,
                     Name = _record.Name,
+                    ActionName = StepTelemetry?.Action,
+                    Ref = StepTelemetry?.Ref,
+                    Type = StepTelemetry?.Type,
                     StartedAt = _record.StartTime,
                     CompletedAt = _record.FinishTime,
                     Annotations = new List<Annotation>()
@@ -812,11 +815,6 @@ namespace GitHub.Runner.Worker
 
             Global.Variables = new Variables(HostContext, variables);
 
-            if (Global.Variables.GetBoolean("DistributedTask.ForceInternalNodeVersionOnRunnerTo16") ?? false)
-            {
-                Environment.SetEnvironmentVariable(Constants.Variables.Agent.ForcedInternalNodeVersion, "node16");
-            }
-
             // Environment variables shared across all actions
             Global.EnvironmentVariables = new Dictionary<string, string>(VarUtil.EnvironmentVariableKeyComparer);
 

src/Runner.Worker/Handlers/HandlerFactory.cs

@@ -57,72 +57,13 @@ namespace GitHub.Runner.Worker.Handlers
                 handler = HostContext.CreateService<INodeScriptActionHandler>();
                 var nodeData = data as NodeJSActionExecutionData;
 
-                // With node12 EoL in 04/2022, we want to be able to uniformly upgrade all JS actions to node16 from the server
+                // With node12 EoL in 04/2022 and node16 EoL in 09/23, we want to execute all JS actions using node20
-                if (string.Equals(nodeData.NodeVersion, "node12", StringComparison.InvariantCultureIgnoreCase))
+                if (string.Equals(nodeData.NodeVersion, "node12", StringComparison.InvariantCultureIgnoreCase) ||
+                    string.Equals(nodeData.NodeVersion, "node16", StringComparison.InvariantCultureIgnoreCase))
                 {
-                    var repoAction = action as Pipelines.RepositoryPathReference;
+                    nodeData.NodeVersion = "node20";
-                    if (repoAction != null)
-                    {
-                        var warningActions = new HashSet<string>();
-                        if (executionContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode12DetectedAfterEndOfLifeEnvVariable, out var node16ForceWarnings))
-                        {
-                            warningActions = StringUtil.ConvertFromJson<HashSet<string>>(node16ForceWarnings);
-                        }
-
-                        string repoActionFullName;
-                        if (string.IsNullOrEmpty(repoAction.Name))
-                        {
-                            repoActionFullName = repoAction.Path; // local actions don't have a 'Name'
-                        }
-                        else
-                        {
-                            repoActionFullName = $"{repoAction.Name}/{repoAction.Path ?? string.Empty}".TrimEnd('/') + $"@{repoAction.Ref}";
-                        }
-
-                        warningActions.Add(repoActionFullName);
-                        executionContext.Global.Variables.Set("Node16ForceActionsWarnings", StringUtil.ConvertToJson(warningActions));
-                    }
-                    nodeData.NodeVersion = "node16";
                 }
 
-                var localForceActionsToNode20 = StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Agent.ManualForceActionsToNode20));
-                executionContext.Global.EnvironmentVariables.TryGetValue(Constants.Variables.Actions.ManualForceActionsToNode20, out var workflowForceActionsToNode20);
-                var enforceNode20Locally = !string.IsNullOrWhiteSpace(workflowForceActionsToNode20) ? StringUtil.ConvertToBoolean(workflowForceActionsToNode20) : localForceActionsToNode20;
-                if (string.Equals(nodeData.NodeVersion, "node16")
-                && ((executionContext.Global.Variables.GetBoolean("DistributedTask.ForceGithubJavascriptActionsToNode20") ?? false) || enforceNode20Locally))
-                {
-                    executionContext.Global.EnvironmentVariables.TryGetValue(Constants.Variables.Actions.AllowActionsUseUnsecureNodeVersion, out var workflowOptOut);
-                    var isWorkflowOptOutSet = !string.IsNullOrWhiteSpace(workflowOptOut);
-                    var isLocalOptOut = StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Actions.AllowActionsUseUnsecureNodeVersion));
-                    bool isOptOut = isWorkflowOptOutSet ? StringUtil.ConvertToBoolean(workflowOptOut) : isLocalOptOut;
-
-                    if (!isOptOut)
-                    {
-                        var repoAction = action as Pipelines.RepositoryPathReference;
-                        if (repoAction != null)
-                        {
-                            var warningActions = new HashSet<string>();
-                            if (executionContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings))
-                            {
-                                warningActions = StringUtil.ConvertFromJson<HashSet<string>>(node20ForceWarnings);
-                            }
-
-                            string repoActionFullName;
-                            if (string.IsNullOrEmpty(repoAction.Name))
-                            {
-                                repoActionFullName = repoAction.Path; // local actions don't have a 'Name'
-                            }
-                            else
-                            {
-                                repoActionFullName = $"{repoAction.Name}/{repoAction.Path ?? string.Empty}".TrimEnd('/') + $"@{repoAction.Ref}";
-                            }
-
-                            warningActions.Add(repoActionFullName);
-                            executionContext.Global.Variables.Set(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, StringUtil.ConvertToJson(warningActions));
-                        }
-                        nodeData.NodeVersion = "node20";
-                    }
-                }
                 (handler as INodeScriptActionHandler).Data = nodeData;
             }
             else if (data.ExecutionType == ActionExecutionType.Script)

src/Runner.Worker/Handlers/NodeScriptActionHandler.cs

@@ -72,6 +72,11 @@ namespace GitHub.Runner.Worker.Handlers
                 Environment["ACTIONS_RESULTS_URL"] = resultsUrl;
             }
 
+            if (ExecutionContext.Global.Variables.GetBoolean("actions_uses_cache_service_v2") ?? false)
+            {
+                Environment["ACTIONS_CACHE_SERVICE_V2"] = bool.TrueString;
+            }
+
             // Resolve the target script.
             string target = null;
             if (stage == ActionRunStage.Main)
@@ -105,23 +110,6 @@ namespace GitHub.Runner.Worker.Handlers
                 workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
             }
 
-            if (string.Equals(Data.NodeVersion, "node12", StringComparison.OrdinalIgnoreCase) &&
-                Constants.Runner.PlatformArchitecture.Equals(Constants.Architecture.Arm64))
-            {
-                ExecutionContext.Output($"The node12 is not supported. Use node16 instead.");
-                Data.NodeVersion = "node16";
-            }
-
-            string forcedNodeVersion = System.Environment.GetEnvironmentVariable(Constants.Variables.Agent.ForcedActionsNodeVersion);
-            if (forcedNodeVersion == "node16" && Data.NodeVersion != "node16")
-            {
-                Data.NodeVersion = "node16";
-            }
-
-            if (forcedNodeVersion == "node20" && Data.NodeVersion != "node20")
-            {
-                Data.NodeVersion = "node20";
-            }
             var nodeRuntimeVersion = await StepHost.DetermineNodeRuntimeVersion(ExecutionContext, Data.NodeVersion);
             ExecutionContext.StepTelemetry.Type = nodeRuntimeVersion;
             string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), nodeRuntimeVersion, "bin", $"node{IOUtil.ExeExtension}");
@@ -143,28 +131,6 @@ namespace GitHub.Runner.Worker.Handlers
             // Remove environment variable that may cause conflicts with the node within the runner.
             Environment.Remove("NODE_ICU_DATA"); // https://github.com/actions/runner/issues/795
 
-            if (string.Equals(Data.NodeVersion, Constants.Runner.DeprecatedNodeVersion, StringComparison.OrdinalIgnoreCase) && (ExecutionContext.Global.Variables.GetBoolean(Constants.Runner.Features.Node16Warning) ?? false))
-            {
-                var repoAction = Action as RepositoryPathReference;
-                var warningActions = new HashSet<string>();
-                if (ExecutionContext.Global.Variables.TryGetValue(Constants.Runner.DeprecatedNodeDetectedAfterEndOfLifeActions, out var deprecatedNodeWarnings))
-                {
-                    warningActions = StringUtil.ConvertFromJson<HashSet<string>>(deprecatedNodeWarnings);
-                }
-
-                if (string.IsNullOrEmpty(repoAction.Name))
-                {
-                    // local actions don't have a 'Name'
-                    warningActions.Add(repoAction.Path);
-                }
-                else
-                {
-                    warningActions.Add($"{repoAction.Name}/{repoAction.Path ?? string.Empty}".TrimEnd('/') + $"@{repoAction.Ref}");
-                }
-
-                ExecutionContext.Global.Variables.Set(Constants.Runner.DeprecatedNodeDetectedAfterEndOfLifeActions, StringUtil.ConvertToJson(warningActions));
-            }
-
             using (var stdoutManager = new OutputManager(ExecutionContext, ActionCommandManager))
             using (var stderrManager = new OutputManager(ExecutionContext, ActionCommandManager))
             {

src/Runner.Worker/JobExtension.cs

@@ -17,6 +17,7 @@ using GitHub.Runner.Common;
 using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
+using Newtonsoft.Json;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
 namespace GitHub.Runner.Worker
@@ -42,11 +43,13 @@ namespace GitHub.Runner.Worker
     public sealed class JobExtension : RunnerService, IJobExtension
     {
         private readonly HashSet<string> _existingProcesses = new(StringComparer.OrdinalIgnoreCase);
-        private readonly List<Task<string>> _connectivityCheckTasks = new();
+        private readonly List<Task<CheckResult>> _connectivityCheckTasks = new();
         private bool _processCleanup;
         private string _processLookupId = $"github_{Guid.NewGuid()}";
         private CancellationTokenSource _diskSpaceCheckToken = new();
         private Task _diskSpaceCheckTask = null;
+        private CancellationTokenSource _serviceConnectivityCheckToken = new();
+        private Task _serviceConnectivityCheckTask = null;
 
         // Download all required actions.
         // Make sure all condition inputs are valid.
@@ -127,10 +130,6 @@ namespace GitHub.Runner.Worker
                         }
                     }
 
-                    // Check OS warning
-                    var osWarningChecker = HostContext.GetService<IOSWarningChecker>();
-                    await osWarningChecker.CheckOSAsync(context);
-
                     try
                     {
                         var tokenPermissions = jobContext.Global.Variables.Get("system.github.token.permissions") ?? "";
@@ -458,11 +457,14 @@ namespace GitHub.Runner.Worker
                         {
                             foreach (var checkUrl in checkUrls)
                             {
-                                _connectivityCheckTasks.Add(CheckConnectivity(checkUrl));
+                                _connectivityCheckTasks.Add(CheckConnectivity(checkUrl, accessToken: string.Empty, timeoutInSeconds: 5, token: CancellationToken.None));
                             }
                         }
                     }
 
+                    Trace.Info($"Start checking service connectivity in background.");
+                    _serviceConnectivityCheckTask = CheckServiceConnectivityAsync(context, _serviceConnectivityCheckToken.Token);
+
                     return steps;
                 }
                 catch (OperationCanceledException ex) when (jobContext.CancellationToken.IsCancellationRequested)
@@ -696,7 +698,7 @@ namespace GitHub.Runner.Worker
                             {
                                 var result = await check;
                                 Trace.Info($"Connectivity check result: {result}");
-                                context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = result });
+                                context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = $"{result.EndpointUrl}: {result.StatusCode}" });
                             }
                         }
                         catch (Exception ex)
@@ -706,6 +708,22 @@ namespace GitHub.Runner.Worker
                             context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = $"Fail to check server connectivity. {ex.Message}" });
                         }
                     }
+
+                    // Collect service connectivity check result
+                    if (_serviceConnectivityCheckTask != null)
+                    {
+                        _serviceConnectivityCheckToken.Cancel();
+                        try
+                        {
+                            await _serviceConnectivityCheckTask;
+                        }
+                        catch (Exception ex)
+                        {
+                            Trace.Error($"Fail to check service connectivity.");
+                            Trace.Error(ex);
+                            context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = $"Fail to check service connectivity. {ex.Message}" });
+                        }
+                    }
                 }
                 catch (Exception ex)
                 {
@@ -721,11 +739,13 @@ namespace GitHub.Runner.Worker
             }
         }
 
-        private async Task<string> CheckConnectivity(string endpointUrl)
+        private async Task<CheckResult> CheckConnectivity(string endpointUrl, string accessToken, int timeoutInSeconds, CancellationToken token)
         {
             Trace.Info($"Check server connectivity for {endpointUrl}.");
-            string result = string.Empty;
+            CheckResult result = new CheckResult() { EndpointUrl = endpointUrl };
-            using (var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)))
+            var stopwatch = Stopwatch.StartNew();
+            using (var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutInSeconds)))
+            using (var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutTokenSource.Token))
             {
                 try
                 {
@@ -733,21 +753,44 @@ namespace GitHub.Runner.Worker
                     using (var httpClient = new HttpClient(httpClientHandler))
                     {
                         httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
-                        var response = await httpClient.GetAsync(endpointUrl, timeoutTokenSource.Token);
+                        if (!string.IsNullOrEmpty(accessToken))
-                        result = $"{endpointUrl}: {response.StatusCode}";
+                        {
+                            httpClient.DefaultRequestHeaders.Add("Authorization", $"Bearer {accessToken}");
+                        }
+
+                        var response = await httpClient.GetAsync(endpointUrl, linkedTokenSource.Token);
+                        result.StatusCode = $"{response.StatusCode}";
+
+                        var githubRequestId = UrlUtil.GetGitHubRequestId(response.Headers);
+                        var vssRequestId = UrlUtil.GetVssRequestId(response.Headers);
+                        if (!string.IsNullOrEmpty(githubRequestId))
+                        {
+                            result.RequestId = githubRequestId;
+                        }
+                        else if (!string.IsNullOrEmpty(vssRequestId))
+                        {
+                            result.RequestId = vssRequestId;
+                        }
                     }
                 }
+                catch (Exception ex) when (ex is OperationCanceledException && token.IsCancellationRequested)
+                {
+                    Trace.Error($"Request canceled during connectivity check: {ex}");
+                    result.StatusCode = "canceled";
+                }
                 catch (Exception ex) when (ex is OperationCanceledException && timeoutTokenSource.IsCancellationRequested)
                 {
                     Trace.Error($"Request timeout during connectivity check: {ex}");
-                    result = $"{endpointUrl}: timeout";
+                    result.StatusCode = "timeout";
                 }
                 catch (Exception ex)
                 {
                     Trace.Error($"Catch exception during connectivity check: {ex}");
-                    result = $"{endpointUrl}: {ex.Message}";
+                    result.StatusCode = $"{ex.Message}";
                 }
             }
+            stopwatch.Stop();
+            result.DurationInMs = (int)stopwatch.ElapsedMilliseconds;
 
             return result;
         }
@@ -785,6 +828,84 @@ namespace GitHub.Runner.Worker
             }
         }
 
+        private async Task CheckServiceConnectivityAsync(IExecutionContext context, CancellationToken token)
+        {
+            var connectionTest = context.Global.Variables.Get(WellKnownDistributedTaskVariables.RunnerServiceConnectivityTest);
+            if (string.IsNullOrEmpty(connectionTest))
+            {
+                return;
+            }
+
+            ServiceConnectivityCheckInput checkConnectivityInfo;
+            try
+            {
+                checkConnectivityInfo = StringUtil.ConvertFromJson<ServiceConnectivityCheckInput>(connectionTest);
+            }
+            catch (Exception ex)
+            {
+                context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $"Fail to parse JSON. {ex.Message}" });
+                return;
+            }
+
+            if (checkConnectivityInfo == null)
+            {
+                return;
+            }
+
+            // make sure interval is at least 10 seconds
+            checkConnectivityInfo.IntervalInSecond = Math.Max(10, checkConnectivityInfo.IntervalInSecond);
+
+            var systemConnection = context.Global.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
+            var accessToken = systemConnection.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken];
+
+            var testResult = new ServiceConnectivityCheckResult();
+            while (!token.IsCancellationRequested)
+            {
+                foreach (var endpoint in checkConnectivityInfo.Endpoints)
+                {
+                    if (string.IsNullOrEmpty(endpoint.Key) || string.IsNullOrEmpty(endpoint.Value))
+                    {
+                        continue;
+                    }
+
+                    if (!testResult.EndpointsResult.ContainsKey(endpoint.Key))
+                    {
+                        testResult.EndpointsResult[endpoint.Key] = new List<string>();
+                    }
+
+                    try
+                    {
+                        var result = await CheckConnectivity(endpoint.Value, accessToken: accessToken, timeoutInSeconds: checkConnectivityInfo.RequestTimeoutInSecond, token);
+                        testResult.EndpointsResult[endpoint.Key].Add($"{result.StartTime:s}: {result.StatusCode} - {result.RequestId} - {result.DurationInMs}ms");
+                        if (!testResult.HasFailure &&
+                            result.StatusCode != "OK" &&
+                            result.StatusCode != "canceled")
+                        {
+                            // track if any endpoint is not reachable
+                            testResult.HasFailure = true;
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        testResult.EndpointsResult[endpoint.Key].Add($"{DateTime.UtcNow:s}: {ex.Message}");
+                    }
+                }
+
+                try
+                {
+                    await Task.Delay(TimeSpan.FromSeconds(checkConnectivityInfo.IntervalInSecond), token);
+                }
+                catch (TaskCanceledException)
+                {
+                    // ignore
+                }
+            }
+
+            var telemetryData = StringUtil.ConvertToJson(testResult, Formatting.None);
+            Trace.Verbose($"Connectivity check result: {telemetryData}");
+            context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = telemetryData });
+        }
+
         private Dictionary<int, Process> SnapshotProcesses()
         {
             Dictionary<int, Process> snapshot = new();
@@ -816,5 +937,23 @@ namespace GitHub.Runner.Worker
                 throw new ArgumentException("Jobs without a job container are forbidden on this runner, please add a 'container:' to your job or contact your self-hosted runner administrator.");
             }
         }
+
+        private class CheckResult
+        {
+            public CheckResult()
+            {
+                StartTime = DateTime.UtcNow;
+            }
+
+            public string EndpointUrl { get; set; }
+
+            public DateTime StartTime { get; set; }
+
+            public string StatusCode { get; set; }
+
+            public string RequestId { get; set; }
+
+            public int DurationInMs { get; set; }
+        }
     }
 }

src/Runner.Worker/JobRunner.cs

@@ -15,6 +15,7 @@ using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
 using GitHub.Services.WebApi;
+using Sdk.RSWebApi.Contracts;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
 namespace GitHub.Runner.Worker
@@ -49,7 +50,8 @@ namespace GitHub.Runner.Worker
             if (message.Variables.TryGetValue(Constants.Variables.System.OrchestrationId, out VariableValue orchestrationId) &&
                 !string.IsNullOrEmpty(orchestrationId.Value))
             {
-                HostContext.UserAgents.Add(new ProductInfoHeaderValue("OrchestrationId", orchestrationId.Value));
+                // make the orchestration id the first item in the user-agent header to avoid get truncated in server log.
+                HostContext.UserAgents.Insert(0, new ProductInfoHeaderValue("OrchestrationId", orchestrationId.Value));
 
                 // make sure orchestration id is in the user-agent header.
                 VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy);
@@ -278,26 +280,14 @@ namespace GitHub.Runner.Worker
         {
             jobContext.Debug($"Finishing: {message.JobDisplayName}");
             TaskResult result = jobContext.Complete(taskResult);
-            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.DeprecatedNodeDetectedAfterEndOfLifeActions, out var deprecatedNodeWarnings))
-            {
-                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(deprecatedNodeWarnings));
-                jobContext.Warning(string.Format(Constants.Runner.DetectedNodeAfterEndOfLifeMessage, actions));
-            }
 
-            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode12DetectedAfterEndOfLifeEnvVariable, out var node16ForceWarnings))
+            var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: false);
+            // include any job telemetry from the background upload process.
+            if (jobQueueTelemetry?.Count > 0)
             {
-                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(node16ForceWarnings));
+                jobContext.Global.JobTelemetry.AddRange(jobQueueTelemetry);
-                jobContext.Warning(string.Format(Constants.Runner.EnforcedNode12DetectedAfterEndOfLife, actions));
             }
 
-            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings) && (jobContext.Global.Variables.GetBoolean("DistributedTask.ForceGithubJavascriptActionsToNode20") ?? false))
-            {
-                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(node20ForceWarnings));
-                jobContext.Warning(string.Format(Constants.Runner.EnforcedNode16DetectedAfterEndOfLife, actions));
-            }
-
-            await ShutdownQueue(throwOnFailure: false);
-
             // Make sure to clean temp after file upload since they may be pending fileupload still use the TEMP dir.
             _tempDirectoryManager?.CleanupTempDirectory();
 
@@ -314,6 +304,13 @@ namespace GitHub.Runner.Worker
                 environmentUrl = urlStringToken.Value;
             }
 
+            // Get telemetry
+            IList<Telemetry> telemetry = null;
+            if (jobContext.Global.JobTelemetry.Count > 0)
+            {
+                telemetry = jobContext.Global.JobTelemetry.Select(x => new Telemetry { Type = x.Type.ToString(), Message = x.Message, }).ToList();
+            }
+
             Trace.Info($"Raising job completed against run service");
             var completeJobRetryLimit = 5;
             var exceptions = new List<Exception>();
@@ -321,9 +318,23 @@ namespace GitHub.Runner.Worker
             {
                 try
                 {
-                    await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, default);
+                    await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, telemetry, billingOwnerId: message.BillingOwnerId, default);
                     return result;
                 }
+                catch (VssUnauthorizedException ex)
+                {
+                    Trace.Error($"Catch exception while attempting to complete job {message.JobId}, job request {message.RequestId}.");
+                    Trace.Error(ex);
+                    exceptions.Add(ex);
+                    break;
+                }
+                catch (TaskOrchestrationJobNotFoundException ex)
+                {
+                    Trace.Error($"Catch exception while attempting to complete job {message.JobId}, job request {message.RequestId}.");
+                    Trace.Error(ex);
+                    exceptions.Add(ex);
+                    break;
+                }
                 catch (Exception ex)
                 {
                     Trace.Error($"Catch exception while attempting to complete job {message.JobId}, job request {message.RequestId}.");
@@ -346,74 +357,14 @@ namespace GitHub.Runner.Worker
 
             if (_runnerSettings.DisableUpdate == true)
             {
-                try
+                await WarningOutdatedRunnerAsync(jobContext, message, result);
-                {
-                    var currentVersion = new PackageVersion(BuildConstants.RunnerPackage.Version);
-                    ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
-                    VssCredentials serverCredential = VssUtil.GetVssCredential(systemConnection);
-
-                    var runnerServer = HostContext.GetService<IRunnerServer>();
-                    await runnerServer.ConnectAsync(systemConnection.Url, serverCredential);
-                    var serverPackages = await runnerServer.GetPackagesAsync("agent", BuildConstants.RunnerPackage.PackageName, 5, includeToken: false, cancellationToken: CancellationToken.None);
-                    if (serverPackages.Count > 0)
-                    {
-                        serverPackages = serverPackages.OrderByDescending(x => x.Version).ToList();
-                        Trace.Info($"Newer packages {StringUtil.ConvertToJson(serverPackages.Select(x => x.Version.ToString()))}");
-
-                        var warnOnFailedJob = false; // any minor/patch version behind.
-                        var warnOnOldRunnerVersion = false; // >= 2 minor version behind
-                        if (serverPackages.Any(x => x.Version.CompareTo(currentVersion) > 0))
-                        {
-                            Trace.Info($"Current runner version {currentVersion} is behind the latest runner version {serverPackages[0].Version}.");
-                            warnOnFailedJob = true;
-                        }
-
-                        if (serverPackages.Where(x => x.Version.Major == currentVersion.Major && x.Version.Minor > currentVersion.Minor).Count() > 1)
-                        {
-                            Trace.Info($"Current runner version {currentVersion} is way behind the latest runner version {serverPackages[0].Version}.");
-                            warnOnOldRunnerVersion = true;
-                        }
-
-                        if (result == TaskResult.Failed && warnOnFailedJob)
-                        {
-                            jobContext.Warning($"This job failure may be caused by using an out of date self-hosted runner. You are currently using runner version {currentVersion}. Please update to the latest version {serverPackages[0].Version}");
-                        }
-                        else if (warnOnOldRunnerVersion)
-                        {
-                            jobContext.Warning($"This self-hosted runner is currently using runner version {currentVersion}. This version is out of date. Please update to the latest version {serverPackages[0].Version}");
-                        }
-                    }
-                }
-                catch (Exception ex)
-                {
-                    // Ignore any error since suggest runner update is best effort.
-                    Trace.Error($"Caught exception during runner version check: {ex}");
-                }
-            }
-
-            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.DeprecatedNodeDetectedAfterEndOfLifeActions, out var deprecatedNodeWarnings))
-            {
-                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(deprecatedNodeWarnings));
-                jobContext.Warning(string.Format(Constants.Runner.DetectedNodeAfterEndOfLifeMessage, actions));
-            }
-
-            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode12DetectedAfterEndOfLifeEnvVariable, out var node16ForceWarnings))
-            {
-                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(node16ForceWarnings));
-                jobContext.Warning(string.Format(Constants.Runner.EnforcedNode12DetectedAfterEndOfLife, actions));
-            }
-
-            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings))
-            {
-                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(node20ForceWarnings));
-                jobContext.Warning(string.Format(Constants.Runner.EnforcedNode16DetectedAfterEndOfLife, actions));
             }
 
             try
             {
                 var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: true);
                 // include any job telemetry from the background upload process.
-                if (jobQueueTelemetry.Count > 0)
+                if (jobQueueTelemetry?.Count > 0)
                 {
                     jobContext.Global.JobTelemetry.AddRange(jobQueueTelemetry);
                 }
@@ -541,5 +492,52 @@ namespace GitHub.Runner.Worker
 
             return Array.Empty<JobTelemetry>();
         }
+
+        private async Task WarningOutdatedRunnerAsync(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult result)
+        {
+            try
+            {
+                var currentVersion = new PackageVersion(BuildConstants.RunnerPackage.Version);
+                ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
+                VssCredentials serverCredential = VssUtil.GetVssCredential(systemConnection);
+
+                var runnerServer = HostContext.GetService<IRunnerServer>();
+                await runnerServer.ConnectAsync(systemConnection.Url, serverCredential);
+                var serverPackages = await runnerServer.GetPackagesAsync("agent", BuildConstants.RunnerPackage.PackageName, 5, includeToken: false, cancellationToken: CancellationToken.None);
+                if (serverPackages.Count > 0)
+                {
+                    serverPackages = serverPackages.OrderByDescending(x => x.Version).ToList();
+                    Trace.Info($"Newer packages {StringUtil.ConvertToJson(serverPackages.Select(x => x.Version.ToString()))}");
+
+                    var warnOnFailedJob = false; // any minor/patch version behind.
+                    var warnOnOldRunnerVersion = false; // >= 2 minor version behind
+                    if (serverPackages.Any(x => x.Version.CompareTo(currentVersion) > 0))
+                    {
+                        Trace.Info($"Current runner version {currentVersion} is behind the latest runner version {serverPackages[0].Version}.");
+                        warnOnFailedJob = true;
+                    }
+
+                    if (serverPackages.Where(x => x.Version.Major == currentVersion.Major && x.Version.Minor > currentVersion.Minor).Count() > 1)
+                    {
+                        Trace.Info($"Current runner version {currentVersion} is way behind the latest runner version {serverPackages[0].Version}.");
+                        warnOnOldRunnerVersion = true;
+                    }
+
+                    if (result == TaskResult.Failed && warnOnFailedJob)
+                    {
+                        jobContext.Warning($"This job failure may be caused by using an out of date version of GitHub runner on your self-hosted runner. You are currently using GitHub runner version {currentVersion}. Please update to the latest version {serverPackages[0].Version}");
+                    }
+                    else if (warnOnOldRunnerVersion)
+                    {
+                        jobContext.Warning($"This self-hosted runner is currently using runner version {currentVersion}. This version is out of date. Please update to the latest version {serverPackages[0].Version}");
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                // Ignore any error since suggest runner update is best effort.
+                Trace.Error($"Caught exception during runner version check: {ex}");
+            }
+        }
     }
 }

src/Runner.Worker/OSWarningChecker.cs

@@ -1,110 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Text.RegularExpressions;
-using System.Threading;
-using System.Threading.Tasks;
-using GitHub.DistributedTask.WebApi;
-using GitHub.Runner.Common;
-using GitHub.Runner.Sdk;
-
-namespace GitHub.Runner.Worker
-{
-    [ServiceLocator(Default = typeof(OSWarningChecker))]
-    public interface IOSWarningChecker : IRunnerService
-    {
-        Task CheckOSAsync(IExecutionContext context);
-    }
-
-    public sealed class OSWarningChecker : RunnerService, IOSWarningChecker
-    {
-        private static TimeSpan s_regexTimeout = TimeSpan.FromSeconds(1);
-
-        public async Task CheckOSAsync(IExecutionContext context)
-        {
-            ArgUtil.NotNull(context, nameof(context));
-            if (!context.Global.Variables.System_TestDotNet8Compatibility)
-            {
-                return;
-            }
-
-            context.Output("Testing runner upgrade compatibility");
-            List<string> output = new();
-            object outputLock = new();
-            try
-            {
-                using (var process = HostContext.CreateService<IProcessInvoker>())
-                {
-                    process.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stdout)
-                    {
-                        if (!string.IsNullOrEmpty(stdout.Data))
-                        {
-                            lock (outputLock)
-                            {
-                                output.Add(stdout.Data);
-                                Trace.Info(stdout.Data);
-                            }
-                        }
-                    };
-
-                    process.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stderr)
-                    {
-                        if (!string.IsNullOrEmpty(stderr.Data))
-                        {
-                            lock (outputLock)
-                            {
-                                output.Add(stderr.Data);
-                                Trace.Error(stderr.Data);
-                            }
-                        }
-                    };
-
-                    using (var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
-                    {
-                        int exitCode = await process.ExecuteAsync(
-                            workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root),
-                            fileName: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "testDotNet8Compatibility", $"TestDotNet8Compatibility{IOUtil.ExeExtension}"),
-                            arguments: string.Empty,
-                            environment: null,
-                            cancellationToken: cancellationTokenSource.Token);
-
-                        var outputStr = string.Join("\n", output).Trim();
-                        if (exitCode != 0 || !string.Equals(outputStr, "Hello from .NET 8!", StringComparison.Ordinal))
-                        {
-                            var pattern = context.Global.Variables.System_DotNet8CompatibilityOutputPattern;
-                            if (!string.IsNullOrEmpty(pattern))
-                            {
-                                var regex = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, s_regexTimeout);
-                                if (!regex.IsMatch(outputStr))
-                                {
-                                    return;
-                                }
-                            }
-
-                            var warningMessage = context.Global.Variables.System_DotNet8CompatibilityWarning;
-                            if (!string.IsNullOrEmpty(warningMessage))
-                            {
-                                context.Warning(warningMessage);
-                            }
-
-                            context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test failed with exit code '{exitCode}' and output: {GetShortOutput(context, output)}" });
-                        }
-                    }
-                }
-            }
-            catch (Exception ex)
-            {
-                Trace.Error("An error occurred while testing .NET 8 compatibility'");
-                Trace.Error(ex);
-                context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test encountered exception type '{ex.GetType().FullName}', message: '{ex.Message}', process output: '{GetShortOutput(context, output)}'" });
-            }
-        }
-
-        private static string GetShortOutput(IExecutionContext context, List<string> output)
-        {
-            var length = context.Global.Variables.System_DotNet8CompatibilityOutputLength ?? 200;
-            var outputStr = string.Join("\n", output).Trim();
-            return outputStr.Length > length ? string.Concat(outputStr.Substring(0, length), "[...]") : outputStr;
-        }
-    }
-}

src/Runner.Worker/Runner.Worker.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <OutputType>Exe</OutputType>
     <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
     <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
     <Version>$(Version)</Version>
     <PredefinedCulturesOnly>false</PredefinedCulturesOnly>
     <PublishReadyToRunComposite>true</PublishReadyToRunComposite>
@@ -18,9 +19,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
+    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
-    <PackageReference Include="System.ServiceProcess.ServiceController" Version="4.4.0" />
+    <PackageReference Include="System.ServiceProcess.ServiceController" Version="8.0.0" />
-    <PackageReference Include="System.Threading.Channels" Version="4.4.0" />
+    <PackageReference Include="System.Threading.Channels" Version="8.0.0" />
     <PackageReference Include="YamlDotNet.Signed" Version="5.3.0" />
   </ItemGroup>
 

src/Runner.Worker/Variables.cs

@@ -72,16 +72,8 @@ namespace GitHub.Runner.Worker
 
         public bool? Step_Debug => GetBoolean(Constants.Variables.Actions.StepDebug);
 
-        public string System_DotNet8CompatibilityWarning => Get(Constants.Variables.System.DotNet8CompatibilityWarning);
-
-        public string System_DotNet8CompatibilityOutputPattern => Get(Constants.Variables.System.DotNet8CompatibilityOutputPattern);
-
-        public int? System_DotNet8CompatibilityOutputLength => GetInt(Constants.Variables.System.DotNet8CompatibilityOutputLength);
-
         public string System_PhaseDisplayName => Get(Constants.Variables.System.PhaseDisplayName);
 
-        public bool System_TestDotNet8Compatibility => GetBoolean(Constants.Variables.System.TestDotNet8Compatibility) ?? false;
-
         public string Get(string name)
         {
             Variable variable;

src/Sdk/Common/Common/Diagnostics/VssHttpEventSource.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Diagnostics.Tracing;
 using System.Globalization;
+using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Net.Sockets;
@@ -335,7 +336,25 @@ namespace GitHub.Services.Common.Diagnostics
             if (IsEnabled())
             {
                 SetActivityId(activity);
-                HttpRequestStop(response.RequestMessage.GetHttpMethod(), response.RequestMessage.RequestUri.AbsoluteUri, (Int32)response.StatusCode);
+                var requestId = "NoExpectedHeader";
+                if (response.Headers != null)
+                {
+                    if (response.Headers.TryGetValues("x-github-request-id", out var headerValues) && headerValues != null)
+                    {
+                        requestId = headerValues.FirstOrDefault();
+                    }
+                    else if (response.Headers.TryGetValues("x-vss-e2eid", out headerValues) && headerValues != null)
+                    {
+                        requestId = headerValues.FirstOrDefault();
+                    }
+
+                    if (string.IsNullOrEmpty(requestId))
+                    {
+                        requestId = "NoExpectedHeader";
+                    }
+                }
+
+                HttpRequestStop(response.RequestMessage.GetHttpMethod(), response.RequestMessage.RequestUri.AbsoluteUri, (Int32)response.StatusCode, requestId);
             }
         }
 
@@ -747,15 +766,16 @@ namespace GitHub.Services.Common.Diagnostics
             }
         }
 
-        [Event(24, Level = EventLevel.Verbose, Task = Tasks.HttpRequest, Opcode = EventOpcode.Stop, Message = "Finished {0} request to {1} with status code {2}")]
+        [Event(24, Level = EventLevel.Verbose, Task = Tasks.HttpRequest, Opcode = EventOpcode.Stop, Message = "Finished {0} request to {1} with status code {2} ({3})")]
         private void HttpRequestStop(
             VssHttpMethod method,
             String url,
-            Int32 statusCode)
+            Int32 statusCode,
+            String requestId)
         {
             if (IsEnabled())
             {
-                WriteEvent(24, (Int32)method, url, statusCode);
+                WriteEvent(24, (Int32)method, url, statusCode, requestId);
             }
         }
 

src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs

@@ -34,6 +34,7 @@ namespace GitHub.Services.Common
 
         public String PropertyName { get; set; }
 
+        [Obsolete]
         [SecurityCritical]
         public override void GetObjectData(SerializationInfo info, StreamingContext context)
         {

src/Sdk/Common/Common/VssException.cs

@@ -127,6 +127,7 @@ namespace GitHub.Services.Common
             EventId = (int)info.GetValue("m_eventId", typeof(int));
         }
 
+        [Obsolete]
         [SecurityCritical]
         public override void GetObjectData(SerializationInfo info, StreamingContext context)
         {

src/Sdk/DTGenerated/Generated/TaskAgentHttpClientBase.cs

@@ -23,8 +23,8 @@ using System.IO;
 using System.IO.Compression;
 using System.Linq;
 using System.Net.Http;
-using System.Net.Http.Headers;
 using System.Net.Http.Formatting;
+using System.Net.Http.Headers;
 using System.Threading;
 using System.Threading.Tasks;
 using GitHub.Services.Common;
@@ -827,5 +827,36 @@ namespace GitHub.DistributedTask.WebApi
                 userState: userState,
                 cancellationToken: cancellationToken);
         }
+
+        /// <summary>
+        /// [Preview API]
+        /// </summary>
+        /// <param name="agentId"></param>
+        /// <param name="configType"></param>
+        /// <param name="encodedRunnerConfig"></param>
+        /// <param name="userState"></param>
+        /// <param name="cancellationToken">The cancellation token to cancel operation.</param>
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        public virtual Task<string> RefreshRunnerConfigAsync(
+            int agentId,
+            string configType,
+            string encodedRunnerConfig,
+            object userState = null,
+            CancellationToken cancellationToken = default)
+        {
+            HttpMethod httpMethod = new HttpMethod("POST");
+            Guid locationId = new Guid("13b5d709-74aa-470b-a8e9-bf9f3ded3f18");
+            object routeValues = new { agentId = agentId, configType = configType };
+            HttpContent content = new ObjectContent<string>(encodedRunnerConfig, new VssJsonMediaTypeFormatter(true));
+
+            return SendAsync<string>(
+                httpMethod,
+                locationId,
+                routeValues: routeValues,
+                version: new ApiResourceVersion(6.0, 1),
+                userState: userState,
+                cancellationToken: cancellationToken,
+                content: content);
+        }
     }
 }

src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs

@@ -246,6 +246,13 @@ namespace GitHub.DistributedTask.Pipelines
             set;
         }
 
+        [DataMember(EmitDefaultValue = false)]
+        public String BillingOwnerId
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Gets the collection of variables associated with the current context.
         /// </summary>

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/ReferenceNameBuilder.cs

@@ -99,7 +99,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
         {
             if (!NameValidation.IsValid(value, allowHyphens: true) && value.Length < PipelineConstants.MaxNodeNameLength)
             {
-                error = $"The identifier '{value}' is invalid. IDs may only contain alphanumeric characters, '_', and '-'. IDs must start with a letter or '_' and and must be less than {PipelineConstants.MaxNodeNameLength} characters.";
+                error = $"The identifier '{value}' is invalid. IDs may only contain alphanumeric characters, '_', and '-'. IDs must start with a letter or '_' and must be less than {PipelineConstants.MaxNodeNameLength} characters.";
                 return false;
             }
             else if (!m_distinctNames.Add(value))

src/Sdk/DTWebApi/WebApi/ActionDownloadInfo.cs

@@ -9,6 +9,9 @@ namespace GitHub.DistributedTask.WebApi
         [DataMember(EmitDefaultValue = false)]
         public ActionDownloadAuthentication Authentication { get; set; }
 
+        [DataMember(EmitDefaultValue = false)]
+        public ActionDownloadPackageDetails PackageDetails { get; set; }
+
         [DataMember(EmitDefaultValue = false)]
         public string NameWithOwner { get; set; }
 
@@ -37,4 +40,14 @@ namespace GitHub.DistributedTask.WebApi
         [DataMember(EmitDefaultValue = false)]
         public string Token { get; set; }
     }
+
+    [DataContract]
+    public class ActionDownloadPackageDetails 
+    {
+        [DataMember(EmitDefaultValue = false)]
+        public string Version { get; set; }
+
+        [DataMember(EmitDefaultValue = false)]
+        public string ManifestDigest { get; set; }
+    }
 }

src/Sdk/DTWebApi/WebApi/Runner.cs

@@ -19,7 +19,7 @@ namespace GitHub.DistributedTask.WebApi
             }
 
             /// <summary>
-            /// The url to connect to to poll for messages
+            /// The url to connect to poll for messages
             /// </summary> 
             [JsonProperty("server_url")]
             public string ServerUrl

src/Sdk/DTWebApi/WebApi/RunnerRefreshConfigMessage.cs

@@ -0,0 +1,58 @@
+using System;
+using System.Runtime.Serialization;
+using GitHub.Services.WebApi;
+using Newtonsoft.Json;
+
+namespace GitHub.DistributedTask.WebApi
+{
+    [DataContract]
+    public sealed class RunnerRefreshConfigMessage
+    {
+        public static readonly String MessageType = "RunnerRefreshConfig";
+
+        [JsonConstructor]
+        internal RunnerRefreshConfigMessage()
+        {
+        }
+
+        public RunnerRefreshConfigMessage(
+            string runnerQualifiedId,
+            string configType,
+            string serviceType,
+            string configRefreshUrl)
+        {
+            this.RunnerQualifiedId = runnerQualifiedId;
+            this.ConfigType = configType;
+            this.ServiceType = serviceType;
+            this.ConfigRefreshUrl = configRefreshUrl;
+        }
+
+        [DataMember(Name = "runnerQualifiedId")]
+        public String RunnerQualifiedId
+        {
+            get;
+            private set;
+        }
+
+        [DataMember(Name = "configType")]
+        public String ConfigType
+        {
+            get;
+            private set;
+        }
+
+        [DataMember(Name = "serviceType")]
+        public String ServiceType
+        {
+            get;
+            private set;
+        }
+
+        [DataMember(Name = "configRefreshURL")]
+        public String ConfigRefreshUrl
+        {
+            get;
+            private set;
+        }
+    }
+}

src/Sdk/DTWebApi/WebApi/ServiceConnectivityCheck.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using Newtonsoft.Json;
+
+namespace GitHub.DistributedTask.WebApi
+{
+    [DataContract]
+    public class ServiceConnectivityCheckInput
+    {
+        [JsonConstructor]
+        public ServiceConnectivityCheckInput()
+        {
+            Endpoints = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        }
+
+        [DataMember(EmitDefaultValue = false)]
+        public Dictionary<string, string> Endpoints { get; set; }
+
+        [DataMember(EmitDefaultValue = false)]
+        public int IntervalInSecond { get; set; }
+
+        [DataMember(EmitDefaultValue = false)]
+        public int RequestTimeoutInSecond { get; set; }
+    }
+
+    [DataContract]
+    public class ServiceConnectivityCheckResult
+    {
+        [JsonConstructor]
+        public ServiceConnectivityCheckResult()
+        {
+            EndpointsResult = new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase);
+        }
+
+        [DataMember(Order = 1, EmitDefaultValue = true)]
+        public bool HasFailure { get; set; }
+
+        [DataMember(Order = 2, EmitDefaultValue = false)]
+        public Dictionary<string, List<string>> EndpointsResult { get; set; }
+    }
+}

src/Sdk/DTWebApi/WebApi/WellKnownDistributedTaskVariables.cs

@@ -7,5 +7,6 @@ namespace GitHub.DistributedTask.WebApi
         public static readonly String JobId = "system.jobId";
         public static readonly String RunnerLowDiskspaceThreshold = "system.runner.lowdiskspacethreshold";
         public static readonly String RunnerEnvironment = "system.runnerEnvironment";
+        public static readonly String RunnerServiceConnectivityTest = "system.runner.serviceconnectivitycheckinput";
     }
 }

src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs

@@ -10,5 +10,8 @@ namespace GitHub.Actions.RunService.WebApi
 
         [DataMember(Name = "runnerOS", EmitDefaultValue = false)]
         public string RunnerOS { get; set; }
+
+        [DataMember(Name = "billingOwnerId", EmitDefaultValue = false)]
+        public string BillingOwnerId { get; set; }
     }
 }

src/Sdk/RSWebApi/Contracts/Annotation.cs

@@ -11,6 +11,9 @@ namespace Sdk.RSWebApi.Contracts
         [DataMember(Name = "message", EmitDefaultValue = false)]
         public string Message;
 
+        [DataMember(Name = "title", EmitDefaultValue = false)]
+        public string Title;
+
         [DataMember(Name = "rawDetails", EmitDefaultValue = false)]
         public string RawDetails;
 
@@ -31,5 +34,8 @@ namespace Sdk.RSWebApi.Contracts
 
         [DataMember(Name = "endColumn", EmitDefaultValue = false)]
         public long EndColumn;
+
+        [DataMember(Name = "stepNumber", EmitDefaultValue = false)]
+        public long StepNumber;
     }
 }

src/Sdk/RSWebApi/Contracts/BrokerErrorKind.cs

@@ -5,6 +5,8 @@ namespace GitHub.Actions.RunService.WebApi
     [DataContract]
     public class BrokerErrorKind
     {
+        public const string RunnerNotFound = "RunnerNotFound";
         public const string RunnerVersionTooOld = "RunnerVersionTooOld";
+        public const string HostedRunnerDeprovisioned = "HostedRunnerDeprovisioned";
     }
 }

src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs

@@ -27,7 +27,13 @@ namespace GitHub.Actions.RunService.WebApi
         [DataMember(Name = "annotations", EmitDefaultValue = false)]
         public IList<Annotation> Annotations { get; set; }
 
+        [DataMember(Name = "telemetry", EmitDefaultValue = false)]
+        public IList<Telemetry> Telemetry { get; set; }
+
         [DataMember(Name = "environmentUrl", EmitDefaultValue = false)]
         public string EnvironmentUrl { get; set; }
+
+        [DataMember(Name = "billingOwnerId", EmitDefaultValue = false)]
+        public string BillingOwnerId { get; set; }
     }
 }

src/Sdk/RSWebApi/Contracts/IssueExtensions.cs

@@ -22,6 +22,8 @@ namespace Sdk.RSWebApi.Contracts
             var columnNumber = GetAnnotationNumber(issue, RunIssueKeys.Col) ?? 0;
             var endColumnNumber = GetAnnotationNumber(issue, RunIssueKeys.EndColumn) ?? columnNumber;
             var logLineNumber = GetAnnotationNumber(issue, RunIssueKeys.LogLineNumber) ?? 0;
+            var stepNumber = GetAnnotationNumber(issue, RunIssueKeys.StepNumber) ?? 0;
+            var title = GetAnnotationField(issue, RunIssueKeys.Title);
 
             if (path == null && lineNumber == 0 && logLineNumber != 0)
             {
@@ -33,11 +35,13 @@ namespace Sdk.RSWebApi.Contracts
             {
                 Level = annotationLevel,
                 Message = issueMessage,
+                Title = title,
                 Path = path,
                 StartLine = lineNumber,
                 EndLine = endLineNumber,
                 StartColumn = columnNumber,
                 EndColumn = endColumnNumber,
+                StepNumber = stepNumber,
             };
         }
 

src/Sdk/RSWebApi/Contracts/IssueKeys.cs

@@ -9,5 +9,7 @@
         public const string EndLine = "endLine";
         public const string EndColumn = "endColumn";
         public const string LogLineNumber = "logFileLineNumber";
+        public const string StepNumber = "stepNumber";
+        public const string Title = "title";
     }
 }

src/Sdk/RSWebApi/Contracts/StepResult.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Runtime.Serialization;
 using System.Threading.Tasks;
@@ -16,9 +16,20 @@ namespace GitHub.Actions.RunService.WebApi
         [DataMember(Name = "number", EmitDefaultValue = false)]
         public int? Number { get; set; }
 
+        // Example: "Run actions/checkout@v3"
         [DataMember(Name = "name", EmitDefaultValue = false)]
         public string Name { get; set; }
 
+        // Example: "actions/checkout"
+        [DataMember(Name = "action_name", EmitDefaultValue = false)]
+        public string ActionName { get; set; }
+
+        [DataMember(Name = "ref", EmitDefaultValue = false)]
+        public string Ref { get; set; }
+
+        [DataMember(Name = "type", EmitDefaultValue = false)]
+        public string Type { get; set; }
+
         [DataMember(Name = "status")]
         public TimelineRecordState? Status { get; set; }
 

src/Sdk/RSWebApi/Contracts/Telemetry.cs

@@ -0,0 +1,20 @@
+using System.Runtime.Serialization;
+
+namespace Sdk.RSWebApi.Contracts
+{
+    [DataContract]
+    public struct Telemetry
+    {
+        public Telemetry(string message, string type)
+        {
+            Message = message;
+            Type = type;
+        }
+
+        [DataMember(Name = "message", EmitDefaultValue = false)]
+        public string Message { get; set; }
+
+        [DataMember(Name = "type", EmitDefaultValue = false)]
+        public string Type { get; set; }
+    }
+}

src/Sdk/RSWebApi/RunServiceHttpClient.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Threading;
@@ -70,13 +71,15 @@ namespace GitHub.Actions.RunService.WebApi
             Uri requestUri,
             string messageId,
             string runnerOS,
+            string billingOwnerId,
             CancellationToken cancellationToken = default)
         {
             HttpMethod httpMethod = new HttpMethod("POST");
             var payload = new AcquireJobRequest
             {
                 JobMessageId = messageId,
-                RunnerOS = runnerOS
+                RunnerOS = runnerOS,
+                BillingOwnerId = billingOwnerId,
             };
 
             requestUri = new Uri(requestUri, "acquirejob");
@@ -126,6 +129,8 @@ namespace GitHub.Actions.RunService.WebApi
             IList<StepResult> stepResults,
             IList<Annotation> jobAnnotations,
             string environmentUrl,
+            IList<Telemetry> telemetry,
+            string billingOwnerId,
             CancellationToken cancellationToken = default)
         {
             HttpMethod httpMethod = new HttpMethod("POST");
@@ -138,6 +143,8 @@ namespace GitHub.Actions.RunService.WebApi
                 StepResults = stepResults,
                 Annotations = jobAnnotations,
                 EnvironmentUrl = environmentUrl,
+                Telemetry = telemetry,
+                BillingOwnerId = billingOwnerId,
             };
 
             requestUri = new Uri(requestUri, "completejob");
@@ -246,11 +253,12 @@ namespace GitHub.Actions.RunService.WebApi
             return false;
         }
 
-        private static string Truncate(string errorBody)
+        internal static string Truncate(string errorBody)
         {
-            if (errorBody.Length > 100)
+            const int maxLength = 200;
+            if (errorBody.Length > maxLength)
             {
-                return errorBody.Substring(0, 100) + "[truncated]";
+                return errorBody.Substring(0, maxLength) + "[truncated]";
             }
 
             return errorBody;

src/Sdk/Sdk.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net6.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <OutputType>Library</OutputType>
         <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+        <!-- <SelfContained>true</SelfContained> -->
         <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-        <NoWarn>NU1701;NU1603</NoWarn>
+        <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
         <Version>$(Version)</Version>
         <DefineConstants>TRACE</DefineConstants>
         <LangVersion>8.0</LangVersion>
@@ -13,18 +14,19 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Azure.Storage.Blobs" Version="12.19.1" />
+        <PackageReference Include="Azure.Storage.Blobs" Version="12.23.0" />
-        <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
+        <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
         <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-        <PackageReference Include="Microsoft.AspNet.WebApi.Client" Version="5.2.9" />
+        <PackageReference Include="Microsoft.AspNet.WebApi.Client" Version="6.0.0" />
-        <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="5.2.1" />
+        <PackageReference Include="System.Security.Cryptography.Cng" Version="5.0.0" />
-        <PackageReference Include="System.Security.Cryptography.Cng" Version="4.4.0" />
+        <PackageReference Include="System.Security.Cryptography.Pkcs" Version="8.0.0" />
-        <PackageReference Include="System.Security.Cryptography.Pkcs" Version="4.4.0" />
+        <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
-        <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
         <PackageReference Include="Minimatch" Version="2.0.0" />
         <PackageReference Include="YamlDotNet.Signed" Version="5.3.0" />
         <PackageReference Include="System.Net.Http" Version="4.3.4" />
         <PackageReference Include="System.Text.RegularExpressions" Version="4.3.1" />
+        <PackageReference Include="System.Private.Uri" Version="4.3.2" />
+        <PackageReference Include="System.Formats.Asn1" Version="8.0.1" />
     </ItemGroup>
 
     <ItemGroup>

src/Sdk/WebApi/WebApi/BrokerHttpClient.cs

@@ -115,11 +115,15 @@ namespace GitHub.Actions.RunService.WebApi
             {
                 switch (brokerError.ErrorKind)
                 {
+                    case BrokerErrorKind.RunnerNotFound:
+                        throw new RunnerNotFoundException(brokerError.Message);
                     case BrokerErrorKind.RunnerVersionTooOld:
                         throw new AccessDeniedException(brokerError.Message)
                         {
                             ErrorCode = 1
                         };
+                    case BrokerErrorKind.HostedRunnerDeprovisioned:
+                        throw new HostedRunnerDeprovisionedException(brokerError.Message);
                     default:
                         break;
                 }

src/Sdk/WebApi/WebApi/Exceptions/HostedRunnerDeprovisionedException.cs

@@ -0,0 +1,23 @@
+using System;
+
+namespace GitHub.Services.WebApi
+{
+    [Serializable]
+    public sealed class HostedRunnerDeprovisionedException : Exception
+    {
+        public HostedRunnerDeprovisionedException()
+            : base()
+        {
+        }
+
+        public HostedRunnerDeprovisionedException(String message)
+            : base(message)
+        {
+        }
+
+        public HostedRunnerDeprovisionedException(String message, Exception innerException)
+            : base(message, innerException)
+        {
+        }
+    }
+}

src/Sdk/WebApi/WebApi/Exceptions/RunnerNotFoundException.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Diagnostics.CodeAnalysis;
+using GitHub.Services.Common;
+using GitHub.Services.WebApi;
+
+namespace GitHub.Services.WebApi
+{
+    [Serializable]
+    public sealed class RunnerNotFoundException : Exception
+    {
+        public RunnerNotFoundException()
+            : base()
+        {
+        }
+
+        public RunnerNotFoundException(String message)
+            : base(message)
+        {
+        }
+
+        public RunnerNotFoundException(String message, Exception innerException)
+            : base(message, innerException)
+        {
+        }
+    }
+}

src/Sdk/WebApi/WebApi/Jwt/JsonWebToken.cs

@@ -25,7 +25,10 @@ namespace GitHub.Services.WebApi.Jwt
         HS256,
 
         [EnumMember]
-        RS256
+        RS256,
+
+        [EnumMember]
+        PS256,
     }
 
     //JsonWebToken is marked as DataContract so
@@ -286,6 +289,7 @@ namespace GitHub.Services.WebApi.Jwt
             {
                 case JWTAlgorithm.HS256:
                 case JWTAlgorithm.RS256:
+                case JWTAlgorithm.PS256:
                     return signingCredentials.SignData(bytes);
 
                 default:

src/Sdk/WebApi/WebApi/LaunchContracts.cs

@@ -29,6 +29,9 @@ namespace GitHub.Services.Launch.Contracts
     {
         [DataMember(EmitDefaultValue = false, Name = "authentication")]
         public ActionDownloadAuthenticationResponse Authentication { get; set; }
+        
+        [DataMember(EmitDefaultValue = false, Name = "package_details")]
+        public ActionDownloadPackageDetailsResponse PackageDetails { get; set; }
 
         [DataMember(EmitDefaultValue = false, Name = "name")]
         public string Name { get; set; }
@@ -59,6 +62,17 @@ namespace GitHub.Services.Launch.Contracts
         public string Token { get; set; }
     }
 
+
+    [DataContract]
+    public class ActionDownloadPackageDetailsResponse 
+    {
+        [DataMember(EmitDefaultValue = false, Name = "version")]
+        public string Version { get; set; }
+
+        [DataMember(EmitDefaultValue = false, Name = "manifest_digest")]
+        public string ManifestDigest { get; set; }
+    }
+
     [DataContract]
     public class ActionDownloadInfoResponseCollection
     {

src/Sdk/WebApi/WebApi/LaunchHttpClient.cs

@@ -91,6 +91,7 @@ namespace GitHub.Services.Launch.Client
                 TarballUrl = actionDownloadInfoResponse.TarUrl,
                 Ref = actionDownloadInfoResponse.Version,
                 ZipballUrl = actionDownloadInfoResponse.ZipUrl,
+                PackageDetails = ToServerData(actionDownloadInfoResponse.PackageDetails)
             };
         }
 
@@ -108,6 +109,21 @@ namespace GitHub.Services.Launch.Client
             };
         }
 
+
+        private static ActionDownloadPackageDetails? ToServerData(ActionDownloadPackageDetailsResponse? actionDownloadPackageDetails)
+        {
+            if (actionDownloadPackageDetails == null)
+            {
+                return null;
+            }
+
+            return new ActionDownloadPackageDetails
+            {
+                Version = actionDownloadPackageDetails.Version,
+                ManifestDigest = actionDownloadPackageDetails.ManifestDigest
+            };
+        }
+
         private MediaTypeFormatter m_formatter;
         private Uri m_launchServiceUrl;
         private string m_token;

src/Sdk/WebApi/WebApi/Location/ServerDataProvider.cs

@@ -677,7 +677,7 @@ namespace GitHub.Services.WebApi.Location
             Int32 lastChangeId = m_locationDataCacheManager.GetLastChangeId();
 
             // If we have -1 then that means we have no disk cache yet or it means that we recently hit an exception trying to reload
-            // the the cache from disk (see Exception catch block in EnsureDiskCacheLoaded).
+            // the cache from disk (see Exception catch block in EnsureDiskCacheLoaded).
             // Either way, we cannot make a call to the server with -1 and pass None.
             // If we do, the resulting payload (which would have ClientCacheFresh=false but include no ServiceDefinitions)
             // would leave the in-memory cache in an inconsistent state

src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs

@@ -85,6 +85,7 @@ namespace GitHub.Services.OAuth
             set;
         }
 
+        [Obsolete]
         public override void GetObjectData(SerializationInfo info, StreamingContext context)
         {
             base.GetObjectData(info, context);

src/Sdk/WebApi/WebApi/ResultsHttpClient.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
@@ -132,7 +133,7 @@ namespace GitHub.Services.Results.Client
 
         private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, Guid stepId, long size, CancellationToken cancellationToken)
         {
-            var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat);
+            var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat, CultureInfo.InvariantCulture);
             var request = new StepSummaryMetadataCreate()
             {
                 WorkflowJobRunBackendId = jobId,
@@ -148,7 +149,7 @@ namespace GitHub.Services.Results.Client
 
         private async Task StepLogUploadCompleteAsync(string planId, string jobId, Guid stepId, long lineCount, CancellationToken cancellationToken)
         {
-            var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat);
+            var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat, CultureInfo.InvariantCulture);
             var request = new StepLogsMetadataCreate()
             {
                 WorkflowJobRunBackendId = jobId,
@@ -164,7 +165,7 @@ namespace GitHub.Services.Results.Client
 
         private async Task JobLogUploadCompleteAsync(string planId, string jobId, long lineCount, CancellationToken cancellationToken)
         {
-            var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat);
+            var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat, CultureInfo.InvariantCulture);
             var request = new JobLogsMetadataCreate()
             {
                 WorkflowJobRunBackendId = jobId,
@@ -565,7 +566,7 @@ namespace GitHub.Services.Results.Client
 
         public async Task UpdateWorkflowStepsAsync(Guid planId, IEnumerable<TimelineRecord> records, CancellationToken cancellationToken)
         {
-            var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat);
+            var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat, CultureInfo.InvariantCulture);
             var stepRecords = records.Where(r => String.Equals(r.RecordType, "Task", StringComparison.Ordinal));
             var stepUpdateRequests = stepRecords.GroupBy(r => r.ParentId).Select(sg => new StepsUpdateRequest()
             {

src/Sdk/WebApi/WebApi/VssServiceResponseException.cs

@@ -24,6 +24,7 @@ namespace GitHub.Services.WebApi
             HttpStatusCode = (HttpStatusCode)info.GetInt32("HttpStatusCode");
         }
 
+        [Obsolete]
         [SecurityCritical]
         public override void GetObjectData(SerializationInfo info, StreamingContext context)
         {

src/Sdk/WebApi/WebApi/VssSigningCredentials.cs

@@ -166,6 +166,21 @@ namespace GitHub.Services.WebApi
                 }
             }
 
+            public override JWTAlgorithm SignatureAlgorithm
+            {
+                get
+                {
+                    if (m_signaturePadding == RSASignaturePadding.Pss)
+                    {
+                        return JWTAlgorithm.PS256;
+                    }
+                    else
+                    {
+                        return base.SignatureAlgorithm;
+                    }
+                }
+            }
+
             protected override Byte[] GetSignature(Byte[] input)
             {
                 using (var rsa = m_factory())

src/Test/L0/HostContextL0.cs

@@ -1,10 +1,10 @@
-using GitHub.Runner.Common.Util;
+using System;
-using System;
 using System.IO;
 using System.Reflection;
 using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading;
+using System.Threading.Tasks;
 using Xunit;
 
 namespace GitHub.Runner.Common.Tests
@@ -172,6 +172,133 @@ namespace GitHub.Runner.Common.Tests
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Common")]
+        public void AuthMigrationDisabledByDefault()
+        {
+            try
+            {
+                Environment.SetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL", "100");
+
+                // Arrange.
+                Setup();
+
+                // Assert.
+                Assert.False(_hc.AllowAuthMigration);
+
+                // Change migration state is error free.
+                _hc.EnableAuthMigration("L0Test");
+                _hc.DeferAuthMigration(TimeSpan.FromHours(1), "L0Test");
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL", null);
+                // Cleanup.
+                Teardown();
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Common")]
+        public async Task AuthMigrationReenableTaskNotRunningByDefault()
+        {
+            try
+            {
+                Environment.SetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL", "50");
+
+                // Arrange.
+                Setup();
+
+                // Assert.
+                Assert.False(_hc.AllowAuthMigration);
+                await Task.Delay(TimeSpan.FromMilliseconds(200));
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL", null);
+                // Cleanup.
+                Teardown();
+            }
+
+            var logFile = Path.Combine(Path.GetDirectoryName(Assembly.GetEntryAssembly().Location), $"trace_{nameof(HostContextL0)}_{nameof(AuthMigrationReenableTaskNotRunningByDefault)}.log");
+            var logContent = await File.ReadAllTextAsync(logFile);
+            Assert.Contains("HostContext", logContent);
+            Assert.DoesNotContain("Auth migration defer timer", logContent);
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Common")]
+        public void AuthMigrationEnableDisable()
+        {
+            try
+            {
+                // Arrange.
+                Setup();
+
+                var eventFiredCount = 0;
+                _hc.AuthMigrationChanged += (sender, e) =>
+                {
+                    eventFiredCount++;
+                    Assert.Equal("L0Test", e.Trace);
+                };
+
+                // Assert.
+                _hc.EnableAuthMigration("L0Test");
+                Assert.True(_hc.AllowAuthMigration);
+
+                _hc.DeferAuthMigration(TimeSpan.FromHours(1), "L0Test");
+                Assert.False(_hc.AllowAuthMigration);
+                Assert.Equal(2, eventFiredCount);
+            }
+            finally
+            {
+                // Cleanup.
+                Teardown();
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Common")]
+        public async Task AuthMigrationAutoReset()
+        {
+            try
+            {
+                Environment.SetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL", "100");
+
+                // Arrange.
+                Setup();
+
+                var eventFiredCount = 0;
+                _hc.AuthMigrationChanged += (sender, e) =>
+                {
+                    eventFiredCount++;
+                    Assert.NotEmpty(e.Trace);
+                };
+
+                // Assert.
+                _hc.EnableAuthMigration("L0Test");
+                Assert.True(_hc.AllowAuthMigration);
+
+                _hc.DeferAuthMigration(TimeSpan.FromMilliseconds(500), "L0Test");
+                Assert.False(_hc.AllowAuthMigration);
+
+                await Task.Delay(TimeSpan.FromSeconds(1));
+                Assert.True(_hc.AllowAuthMigration);
+                Assert.Equal(3, eventFiredCount);
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL", null);
+
+                // Cleanup.
+                Teardown();
+            }
+        }
+
         private void Setup([CallerMemberName] string testName = "")
         {
             _tokenSource = new CancellationTokenSource();

src/Test/L0/Listener/BrokerMessageListenerL0.cs

@@ -1,4 +1,5 @@
 using System;
+using System.IO;
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
@@ -16,9 +17,8 @@ namespace GitHub.Runner.Common.Tests.Listener
         private readonly RunnerSettings _settings;
         private readonly Mock<IConfigurationManager> _config;
         private readonly Mock<IBrokerServer> _brokerServer;
+        private readonly Mock<IRunnerServer> _runnerServer;
         private readonly Mock<ICredentialManager> _credMgr;
-        private Mock<IConfigurationStore> _store;
-
 
         public BrokerMessageListenerL0()
         {
@@ -26,14 +26,14 @@ namespace GitHub.Runner.Common.Tests.Listener
             _config = new Mock<IConfigurationManager>();
             _config.Setup(x => x.LoadSettings()).Returns(_settings);
             _credMgr = new Mock<ICredentialManager>();
-            _store = new Mock<IConfigurationStore>();
             _brokerServer = new Mock<IBrokerServer>();
+            _runnerServer = new Mock<IRunnerServer>();
         }
 
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Runner")]
-        public async void CreatesSession()
+        public async Task CreatesSession()
         {
             using (TestHostContext tc = CreateTestContext())
             using (var tokenSource = new CancellationTokenSource())
@@ -48,9 +48,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials());
-                _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
-                _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 
                 // Act.
                 BrokerMessageListener listener = new();
@@ -68,13 +66,310 @@ namespace GitHub.Runner.Common.Tests.Listener
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async Task HandleAuthMigrationChanged()
+        {
+            using (TestHostContext tc = CreateTestContext())
+            using (var tokenSource = new CancellationTokenSource())
+            {
+                Tracing trace = tc.GetTrace();
+
+                // Arrange.
+                var expectedSession = new TaskAgentSession();
+                _brokerServer
+                    .Setup(x => x.CreateSessionAsync(
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedSession));
+
+                _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials());
+
+                // Act.
+                BrokerMessageListener listener = new();
+                listener.Initialize(tc);
+
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
+                trace.Info("result: {0}", result);
+
+                // Assert.
+                Assert.Equal(CreateSessionResult.Success, result);
+                _brokerServer
+                   .Verify(x => x.CreateSessionAsync(
+                       It.Is<TaskAgentSession>(y => y != null),
+                       tokenSource.Token), Times.Once());
+
+                tc.EnableAuthMigration("L0Test");
+
+                var traceFile = Path.GetTempFileName();
+                File.Copy(tc.TraceFileName, traceFile, true);
+                Assert.Contains("Auth migration changed", File.ReadAllText(traceFile));
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async Task CreatesSession_DeferAuthMigration()
+        {
+            using (TestHostContext tc = CreateTestContext())
+            using (var tokenSource = new CancellationTokenSource())
+            {
+                Tracing trace = tc.GetTrace();
+
+                // Arrange.
+                var throwException = true;
+                var expectedSession = new TaskAgentSession();
+                _brokerServer
+                    .Setup(x => x.CreateSessionAsync(
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(async (TaskAgentSession session, CancellationToken token) =>
+                    {
+                        await Task.Yield();
+                        if (throwException)
+                        {
+                            throwException = false;
+                            throw new NotSupportedException("Error during create session");
+                        }
+
+                        return expectedSession;
+                    });
+
+                _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials());
+
+                // Act.
+                BrokerMessageListener listener = new();
+                listener.Initialize(tc);
+
+                tc.EnableAuthMigration("L0Test");
+                Assert.True(tc.AllowAuthMigration);
+
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
+                trace.Info("result: {0}", result);
+
+                // Assert.
+                Assert.Equal(CreateSessionResult.Success, result);
+                _brokerServer
+                   .Verify(x => x.CreateSessionAsync(
+                       It.Is<TaskAgentSession>(y => y != null),
+                       tokenSource.Token), Times.Exactly(2));
+                _credMgr.Verify(x => x.LoadCredentials(true), Times.Exactly(2));
+
+                Assert.False(tc.AllowAuthMigration);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async Task GetNextMessage()
+        {
+            using (TestHostContext tc = CreateTestContext())
+            using (var tokenSource = new CancellationTokenSource())
+            {
+                Tracing trace = tc.GetTrace();
+
+                // Arrange.
+                _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials());
+
+                var expectedSession = new TaskAgentSession();
+                _brokerServer
+                    .Setup(x => x.CreateSessionAsync(
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedSession));
+
+                var expectedMessage = new TaskAgentMessage();
+                _brokerServer
+                    .Setup(x => x.GetRunnerMessageAsync(
+                        It.IsAny<Guid?>(),
+                        It.IsAny<TaskAgentStatus>(),
+                        It.IsAny<string>(),
+                        It.IsAny<string>(),
+                        It.IsAny<string>(),
+                        It.IsAny<bool>(),
+                        It.IsAny<CancellationToken>()))
+                    .Returns(Task.FromResult(expectedMessage));
+
+                // Act.
+                BrokerMessageListener listener = new();
+                listener.Initialize(tc);
+
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
+                trace.Info("result: {0}", result);
+                Assert.Equal(CreateSessionResult.Success, result);
+
+                TaskAgentMessage message = await listener.GetNextMessageAsync(tokenSource.Token);
+                trace.Info("message: {0}", message);
+
+                // Assert.
+                Assert.Equal(expectedMessage, message);
+                _brokerServer
+                   .Verify(x => x.GetRunnerMessageAsync(
+                       It.IsAny<Guid?>(),
+                       It.IsAny<TaskAgentStatus>(),
+                       It.IsAny<string>(),
+                       It.IsAny<string>(),
+                       It.IsAny<string>(),
+                       It.IsAny<bool>(),
+                       It.IsAny<CancellationToken>()), Times.Once());
+
+                _brokerServer.Verify(x => x.ConnectAsync(It.IsAny<Uri>(), It.IsAny<VssCredentials>()), Times.Once());
+
+                _credMgr.Verify(x => x.LoadCredentials(true), Times.Once());
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async Task GetNextMessage_EnableAuthMigration()
+        {
+            using (TestHostContext tc = CreateTestContext())
+            using (var tokenSource = new CancellationTokenSource())
+            {
+                Tracing trace = tc.GetTrace();
+
+                // Arrange.
+                _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials());
+
+                var expectedSession = new TaskAgentSession();
+                _brokerServer
+                    .Setup(x => x.CreateSessionAsync(
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedSession));
+
+                var expectedMessage = new TaskAgentMessage();
+                _brokerServer
+                    .Setup(x => x.GetRunnerMessageAsync(
+                        It.IsAny<Guid?>(),
+                        It.IsAny<TaskAgentStatus>(),
+                        It.IsAny<string>(),
+                        It.IsAny<string>(),
+                        It.IsAny<string>(),
+                        It.IsAny<bool>(),
+                        It.IsAny<CancellationToken>()))
+                    .Returns(Task.FromResult(expectedMessage));
+
+                // Act.
+                BrokerMessageListener listener = new();
+                listener.Initialize(tc);
+
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
+                trace.Info("result: {0}", result);
+                Assert.Equal(CreateSessionResult.Success, result);
+
+                tc.EnableAuthMigration("L0Test");
+
+                TaskAgentMessage message = await listener.GetNextMessageAsync(tokenSource.Token);
+                trace.Info("message: {0}", message);
+
+                // Assert.
+                Assert.Equal(expectedMessage, message);
+                _brokerServer
+                   .Verify(x => x.GetRunnerMessageAsync(
+                       It.IsAny<Guid?>(),
+                       It.IsAny<TaskAgentStatus>(),
+                       It.IsAny<string>(),
+                       It.IsAny<string>(),
+                       It.IsAny<string>(),
+                       It.IsAny<bool>(),
+                       It.IsAny<CancellationToken>()), Times.Once());
+
+                _brokerServer.Verify(x => x.ConnectAsync(It.IsAny<Uri>(), It.IsAny<VssCredentials>()), Times.Exactly(2));
+
+                _credMgr.Verify(x => x.LoadCredentials(true), Times.Exactly(2));
+
+                Assert.True(tc.AllowAuthMigration);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async Task GetNextMessage_AuthMigrationFallback()
+        {
+            using (TestHostContext tc = CreateTestContext())
+            using (var tokenSource = new CancellationTokenSource())
+            {
+                Tracing trace = tc.GetTrace();
+
+                tc.EnableAuthMigration("L0Test");
+
+                // Arrange.
+                _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials());
+
+                var expectedSession = new TaskAgentSession();
+                _brokerServer
+                    .Setup(x => x.CreateSessionAsync(
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedSession));
+
+                var expectedMessage = new TaskAgentMessage();
+                _brokerServer
+                    .Setup(x => x.GetRunnerMessageAsync(
+                        It.IsAny<Guid?>(),
+                        It.IsAny<TaskAgentStatus>(),
+                        It.IsAny<string>(),
+                        It.IsAny<string>(),
+                        It.IsAny<string>(),
+                        It.IsAny<bool>(),
+                        It.IsAny<CancellationToken>()))
+                    .Returns(async (Guid? sessionId, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate, CancellationToken token) =>
+                    {
+                        await Task.Yield();
+                        if (tc.AllowAuthMigration)
+                        {
+                            throw new NotSupportedException("Error during get message");
+                        }
+
+                        return expectedMessage;
+                    });
+
+                // Act.
+                BrokerMessageListener listener = new();
+                listener.Initialize(tc);
+
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
+                trace.Info("result: {0}", result);
+                Assert.Equal(CreateSessionResult.Success, result);
+
+                Assert.True(tc.AllowAuthMigration);
+
+                TaskAgentMessage message = await listener.GetNextMessageAsync(tokenSource.Token);
+                trace.Info("message: {0}", message);
+
+                // Assert.
+                Assert.Equal(expectedMessage, message);
+                _brokerServer
+                   .Verify(x => x.GetRunnerMessageAsync(
+                       It.IsAny<Guid?>(),
+                       It.IsAny<TaskAgentStatus>(),
+                       It.IsAny<string>(),
+                       It.IsAny<string>(),
+                       It.IsAny<string>(),
+                       It.IsAny<bool>(),
+                       It.IsAny<CancellationToken>()), Times.Exactly(2));
+
+                _brokerServer.Verify(x => x.ConnectAsync(It.IsAny<Uri>(), It.IsAny<VssCredentials>()), Times.Exactly(3));
+
+                _credMgr.Verify(x => x.LoadCredentials(true), Times.Exactly(3));
+
+                Assert.False(tc.AllowAuthMigration);
+            }
+        }
+
         private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
         {
             TestHostContext tc = new(this, testName);
             tc.SetSingleton<IConfigurationManager>(_config.Object);
             tc.SetSingleton<ICredentialManager>(_credMgr.Object);
-            tc.SetSingleton<IConfigurationStore>(_store.Object);
             tc.SetSingleton<IBrokerServer>(_brokerServer.Object);
+            tc.SetSingleton<IRunnerServer>(_runnerServer.Object);
             return tc;
         }
     }

src/Test/L0/Listener/Configuration/RunnerCredentialL0.cs

@@ -1,14 +1,18 @@
-using GitHub.Runner.Listener;
+using System.Collections.Generic;
+using System.Security.Cryptography;
+using GitHub.Runner.Listener;
 using GitHub.Runner.Listener.Configuration;
 using GitHub.Services.Common;
 using GitHub.Services.OAuth;
+using Moq;
+using Xunit;
 
 namespace GitHub.Runner.Common.Tests.Listener.Configuration
 {
     public class TestRunnerCredential : CredentialProvider
     {
         public TestRunnerCredential() : base("TEST") { }
-        public override VssCredentials GetVssCredentials(IHostContext context)
+        public override VssCredentials GetVssCredentials(IHostContext context, bool allowAuthUrlV2)
         {
             Tracing trace = context.GetTrace("OuthAccessToken");
             trace.Info("GetVssCredentials()");
@@ -23,4 +27,85 @@ namespace GitHub.Runner.Common.Tests.Listener.Configuration
         {
         }
     }
-}
+
+    public class OAuthCredentialTestsL0
+    {
+        private Mock<IRSAKeyManager> _rsaKeyManager = new Mock<IRSAKeyManager>();
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "OAuthCredential")]
+        public void NotUseAuthV2Url()
+        {
+            using (TestHostContext hc = new(this))
+            {
+                // Arrange.
+                var oauth = new OAuthCredential();
+                oauth.CredentialData = new CredentialData()
+                {
+                    Scheme = Constants.Configuration.OAuth
+                };
+                oauth.CredentialData.Data.Add("clientId", "someClientId");
+                oauth.CredentialData.Data.Add("authorizationUrl", "http://myserver/");
+                oauth.CredentialData.Data.Add("authorizationUrlV2", "http://myserverv2/");
+
+                _rsaKeyManager.Setup(x => x.GetKey()).Returns(RSA.Create(2048));
+                hc.SetSingleton<IRSAKeyManager>(_rsaKeyManager.Object);
+
+                // Act.
+                var cred = oauth.GetVssCredentials(hc, false); // not allow auth v2
+
+                var cred2 = oauth.GetVssCredentials(hc, true); // use auth v2 but hostcontext doesn't
+
+                hc.EnableAuthMigration("L0Test");
+                var cred3 = oauth.GetVssCredentials(hc, false); // not use auth v2 but hostcontext does
+
+                oauth.CredentialData.Data.Remove("authorizationUrlV2");
+                var cred4 = oauth.GetVssCredentials(hc, true); // v2 url is not there
+
+                // Assert.
+                Assert.Equal("http://myserver/", (cred.Federated as VssOAuthCredential).AuthorizationUrl.AbsoluteUri);
+                Assert.Equal("someClientId", (cred.Federated as VssOAuthCredential).ClientCredential.ClientId);
+
+                Assert.Equal("http://myserver/", (cred2.Federated as VssOAuthCredential).AuthorizationUrl.AbsoluteUri);
+                Assert.Equal("someClientId", (cred2.Federated as VssOAuthCredential).ClientCredential.ClientId);
+
+                Assert.Equal("http://myserver/", (cred3.Federated as VssOAuthCredential).AuthorizationUrl.AbsoluteUri);
+                Assert.Equal("someClientId", (cred3.Federated as VssOAuthCredential).ClientCredential.ClientId);
+
+                Assert.Equal("http://myserver/", (cred4.Federated as VssOAuthCredential).AuthorizationUrl.AbsoluteUri);
+                Assert.Equal("someClientId", (cred4.Federated as VssOAuthCredential).ClientCredential.ClientId);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "OAuthCredential")]
+        public void UseAuthV2Url()
+        {
+            using (TestHostContext hc = new(this))
+            {
+                // Arrange.
+                var oauth = new OAuthCredential();
+                oauth.CredentialData = new CredentialData()
+                {
+                    Scheme = Constants.Configuration.OAuth
+                };
+                oauth.CredentialData.Data.Add("clientId", "someClientId");
+                oauth.CredentialData.Data.Add("authorizationUrl", "http://myserver/");
+                oauth.CredentialData.Data.Add("authorizationUrlV2", "http://myserverv2/");
+
+                _rsaKeyManager.Setup(x => x.GetKey()).Returns(RSA.Create(2048));
+                hc.SetSingleton<IRSAKeyManager>(_rsaKeyManager.Object);
+
+                // Act.
+                hc.EnableAuthMigration("L0Test");
+                var cred = oauth.GetVssCredentials(hc, true);
+
+                // Assert.
+                Assert.Equal("http://myserverv2/", (cred.Federated as VssOAuthCredential).AuthorizationUrl.AbsoluteUri);
+                Assert.Equal("someClientId", (cred.Federated as VssOAuthCredential).ClientCredential.ClientId);
+            }
+        }
+    }
+}

src/Test/L0/Listener/JobDispatcherL0.cs

@@ -36,20 +36,23 @@ namespace GitHub.Runner.Common.Tests.Listener
             _configurationStore = new Mock<IConfigurationStore>();
         }
 
-        private Pipelines.AgentJobRequestMessage CreateJobRequestMessage()
+        private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string billingOwnerId = null)
         {
             TaskOrchestrationPlanReference plan = new();
             TimelineReference timeline = null;
             Guid jobId = Guid.NewGuid();
             var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
             result.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();
+            result.BillingOwnerId = billingOwnerId;
             return result;
         }
 
-        [Fact]
+        [Theory]
         [Trait("Level", "L0")]
         [Trait("Category", "Runner")]
-        public async void DispatchesJobRequest()
+        [InlineData(null)]
+        [InlineData("billingOwnerId")]
+        public async void DispatchesJobRequest(string billingOwnerId)
         {
             //Arrange
             using (var hc = new TestHostContext(this))
@@ -65,7 +68,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                 jobDispatcher.Initialize(hc);
 
                 var ts = new CancellationTokenSource();
-                Pipelines.AgentJobRequestMessage message = CreateJobRequestMessage();
+                Pipelines.AgentJobRequestMessage message = CreateJobRequestMessage(billingOwnerId);
                 string strMessage = JsonUtility.ToString(message);
 
                 _processInvoker.Setup(x => x.ExecuteAsync(It.IsAny<String>(), It.IsAny<String>(), "spawnclient 1 2", null, It.IsAny<CancellationToken>()))

src/Test/L0/Listener/MessageListenerL0.cs

@@ -51,7 +51,7 @@ namespace GitHub.Runner.Common.Tests.Listener
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Runner")]
-        public async void CreatesSession()
+        public async Task CreatesSession()
         {
             using (TestHostContext tc = CreateTestContext())
             using (var tokenSource = new CancellationTokenSource())
@@ -67,7 +67,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
                 _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
                 _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 
@@ -95,69 +95,7 @@ namespace GitHub.Runner.Common.Tests.Listener
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Runner")]
-        public async void CreatesSessionWithBrokerMigration()
+        public async Task DeleteSession()
-        {
-            using (TestHostContext tc = CreateTestContext())
-            using (var tokenSource = new CancellationTokenSource())
-            {
-                Tracing trace = tc.GetTrace();
-
-                // Arrange.
-                var expectedSession = new TaskAgentSession()
-                {
-                    OwnerName = "legacy",
-                    BrokerMigrationMessage = new BrokerMigrationMessage(new Uri("https://broker.actions.github.com"))
-                };
-
-                var expectedBrokerSession = new TaskAgentSession()
-                {
-                    OwnerName = "broker"
-                };
-
-                _runnerServer
-                    .Setup(x => x.CreateAgentSessionAsync(
-                        _settings.PoolId,
-                        It.Is<TaskAgentSession>(y => y != null),
-                        tokenSource.Token))
-                    .Returns(Task.FromResult(expectedSession));
-
-                _brokerServer
-                    .Setup(x => x.CreateSessionAsync(
-                        It.Is<TaskAgentSession>(y => y != null),
-                        tokenSource.Token))
-                    .Returns(Task.FromResult(expectedBrokerSession));
-
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
-                _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
-                _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
-
-                // Act.
-                MessageListener listener = new();
-                listener.Initialize(tc);
-
-                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
-                trace.Info("result: {0}", result);
-
-                // Assert.
-                Assert.Equal(CreateSessionResult.Success, result);
-
-                _runnerServer
-                    .Verify(x => x.CreateAgentSessionAsync(
-                        _settings.PoolId,
-                        It.Is<TaskAgentSession>(y => y != null),
-                        tokenSource.Token), Times.Once());
-
-                _brokerServer
-                    .Verify(x => x.CreateSessionAsync(
-                        It.Is<TaskAgentSession>(y => y != null),
-                        tokenSource.Token), Times.Once());
-            }
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Runner")]
-        public async void DeleteSession()
         {
             using (TestHostContext tc = CreateTestContext())
             using (var tokenSource = new CancellationTokenSource())
@@ -177,7 +115,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
                 _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
                 _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 
@@ -204,84 +142,7 @@ namespace GitHub.Runner.Common.Tests.Listener
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Runner")]
-        public async void DeleteSessionWithBrokerMigration()
+        public async Task GetNextMessage()
-        {
-            using (TestHostContext tc = CreateTestContext())
-            using (var tokenSource = new CancellationTokenSource())
-            {
-                Tracing trace = tc.GetTrace();
-
-                // Arrange.
-                var expectedSession = new TaskAgentSession()
-                {
-                    OwnerName = "legacy",
-                    BrokerMigrationMessage = new BrokerMigrationMessage(new Uri("https://broker.actions.github.com"))
-                };
-
-                var expectedBrokerSession = new TaskAgentSession()
-                {
-                    SessionId = Guid.NewGuid(),
-                    OwnerName = "broker"
-                };
-
-                _runnerServer
-                    .Setup(x => x.CreateAgentSessionAsync(
-                        _settings.PoolId,
-                        It.Is<TaskAgentSession>(y => y != null),
-                        tokenSource.Token))
-                    .Returns(Task.FromResult(expectedSession));
-
-                _brokerServer
-                    .Setup(x => x.CreateSessionAsync(
-                        It.Is<TaskAgentSession>(y => y != null),
-                        tokenSource.Token))
-                    .Returns(Task.FromResult(expectedBrokerSession));
-
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
-                _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
-                _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
-
-                // Act.
-                MessageListener listener = new();
-                listener.Initialize(tc);
-
-                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
-                trace.Info("result: {0}", result);
-
-                Assert.Equal(CreateSessionResult.Success, result);
-
-                _runnerServer
-                    .Verify(x => x.CreateAgentSessionAsync(
-                        _settings.PoolId,
-                        It.Is<TaskAgentSession>(y => y != null),
-                        tokenSource.Token), Times.Once());
-
-                _brokerServer
-                    .Verify(x => x.CreateSessionAsync(
-                        It.Is<TaskAgentSession>(y => y != null),
-                        tokenSource.Token), Times.Once());
-
-                _brokerServer
-                    .Setup(x => x.DeleteSessionAsync(It.IsAny<CancellationToken>()))
-                    .Returns(Task.CompletedTask);
-
-                // Act.
-                await listener.DeleteSessionAsync();
-
-
-                //Assert
-                _runnerServer
-                    .Verify(x => x.DeleteAgentSessionAsync(
-                        _settings.PoolId, expectedBrokerSession.SessionId, It.IsAny<CancellationToken>()), Times.Once());
-                _brokerServer
-                    .Verify(x => x.DeleteSessionAsync(It.IsAny<CancellationToken>()), Times.Once());
-            }
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Runner")]
-        public async void GetNextMessage()
         {
             using (TestHostContext tc = CreateTestContext())
             using (var tokenSource = new CancellationTokenSource())
@@ -301,7 +162,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
                 _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
                 _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 
@@ -362,7 +223,7 @@ namespace GitHub.Runner.Common.Tests.Listener
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Runner")]
-        public async void GetNextMessageWithBrokerMigration()
+        public async Task GetNextMessageWithBrokerMigration()
         {
             using (TestHostContext tc = CreateTestContext())
             using (var tokenSource = new CancellationTokenSource())
@@ -382,7 +243,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
                 _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
                 _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 
@@ -462,13 +323,22 @@ namespace GitHub.Runner.Common.Tests.Listener
                 _brokerServer
                     .Verify(x => x.GetRunnerMessageAsync(
                     expectedSession.SessionId, TaskAgentStatus.Online, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()), Times.Exactly(brokerMessages.Length));
+
+                _credMgr
+                    .Verify(x => x.LoadCredentials(true), Times.Exactly(brokerMessages.Length));
+
+                _brokerServer
+                    .Verify(x => x.UpdateConnectionIfNeeded(brokerMigrationMesage.BrokerBaseUrl, It.IsAny<VssCredentials>()), Times.Exactly(brokerMessages.Length));
+
+                _brokerServer
+                    .Verify(x => x.ForceRefreshConnection(It.IsAny<VssCredentials>()), Times.Never);
             }
         }
 
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Runner")]
-        public async void CreateSessionWithOriginalCredential()
+        public async Task CreateSessionWithOriginalCredential()
         {
             using (TestHostContext tc = CreateTestContext())
             using (var tokenSource = new CancellationTokenSource())
@@ -484,7 +354,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
 
                 var originalCred = new CredentialData() { Scheme = Constants.Configuration.OAuth };
                 originalCred.Data["authorizationUrl"] = "https://s.server";
@@ -513,7 +383,7 @@ namespace GitHub.Runner.Common.Tests.Listener
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Runner")]
-        public async void SkipDeleteSession_WhenGetNextMessageGetTaskAgentAccessTokenExpiredException()
+        public async Task SkipDeleteSession_WhenGetNextMessageGetTaskAgentAccessTokenExpiredException()
         {
             using (TestHostContext tc = CreateTestContext())
             using (var tokenSource = new CancellationTokenSource())
@@ -533,7 +403,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
                 _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
                 _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 
@@ -571,5 +441,301 @@ namespace GitHub.Runner.Common.Tests.Listener
                         _settings.PoolId, expectedSession.SessionId, It.IsAny<CancellationToken>()), Times.Never);
             }
         }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async Task HandleAuthMigrationChanged()
+        {
+            using (TestHostContext tc = CreateTestContext())
+            using (var tokenSource = new CancellationTokenSource())
+            {
+                Tracing trace = tc.GetTrace();
+
+                // Arrange.
+                var expectedSession = new TaskAgentSession();
+                _runnerServer
+                    .Setup(x => x.CreateAgentSessionAsync(
+                        _settings.PoolId,
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedSession));
+
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
+
+                // Act.
+                MessageListener listener = new();
+                listener.Initialize(tc);
+
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
+                trace.Info("result: {0}", result);
+
+                // Assert.
+                Assert.Equal(CreateSessionResult.Success, result);
+                _runnerServer
+                    .Verify(x => x.CreateAgentSessionAsync(
+                        _settings.PoolId,
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token), Times.Once());
+                _brokerServer
+                   .Verify(x => x.CreateSessionAsync(
+                       It.Is<TaskAgentSession>(y => y != null),
+                       tokenSource.Token), Times.Never());
+
+                tc.EnableAuthMigration("L0Test");
+
+                var traceFile = Path.GetTempFileName();
+                File.Copy(tc.TraceFileName, traceFile, true);
+                Assert.Contains("Auth migration changed", File.ReadAllText(traceFile));
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async Task GetNextMessageWithBrokerMigration_AuthMigrationFallback()
+        {
+            using (TestHostContext tc = CreateTestContext())
+            using (var tokenSource = new CancellationTokenSource())
+            {
+                Tracing trace = tc.GetTrace();
+
+                // Arrange.
+                var expectedSession = new TaskAgentSession();
+                PropertyInfo sessionIdProperty = expectedSession.GetType().GetProperty("SessionId", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
+                Assert.NotNull(sessionIdProperty);
+                sessionIdProperty.SetValue(expectedSession, Guid.NewGuid());
+
+                _runnerServer
+                    .Setup(x => x.CreateAgentSessionAsync(
+                        _settings.PoolId,
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedSession));
+
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
+                _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
+                _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
+
+                // Act.
+                MessageListener listener = new();
+                listener.Initialize(tc);
+
+                tc.EnableAuthMigration("L0Test");
+
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
+                Assert.Equal(CreateSessionResult.Success, result);
+
+                var brokerMigrationMesage = new BrokerMigrationMessage(new Uri("https://actions.broker.com"));
+
+                var arMessages = new TaskAgentMessage[]
+                {
+                        new TaskAgentMessage
+                        {
+                            Body = JsonUtility.ToString(brokerMigrationMesage),
+                            MessageType = BrokerMigrationMessage.MessageType
+                        },
+                };
+
+                var brokerMessages = new TaskAgentMessage[]
+                {
+                        new TaskAgentMessage
+                        {
+                            Body = "somebody1",
+                            MessageId = 4234,
+                            MessageType = JobRequestMessageTypes.PipelineAgentJobRequest
+                        },
+                        new TaskAgentMessage
+                        {
+                            Body = "somebody2",
+                            MessageId = 4235,
+                            MessageType = JobCancelMessage.MessageType
+                        },
+                        null,  //should be skipped by GetNextMessageAsync implementation
+                        null,
+                        new TaskAgentMessage
+                        {
+                            Body = "somebody3",
+                            MessageId = 4236,
+                            MessageType = JobRequestMessageTypes.PipelineAgentJobRequest
+                        }
+                };
+                var brokerMessageQueue = new Queue<TaskAgentMessage>(brokerMessages);
+
+                _runnerServer
+                    .Setup(x => x.GetAgentMessageAsync(
+                        _settings.PoolId, expectedSession.SessionId, It.IsAny<long?>(), TaskAgentStatus.Online, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
+                    .Returns(async (Int32 poolId, Guid sessionId, Int64? lastMessageId, TaskAgentStatus status, string runnerVersion, string os, string architecture, bool disableUpdate, CancellationToken cancellationToken) =>
+                    {
+                        await Task.Yield();
+                        return arMessages[0]; // always send migration message
+                    });
+
+                var counter = 0;
+                _brokerServer
+                   .Setup(x => x.GetRunnerMessageAsync(
+                       expectedSession.SessionId, TaskAgentStatus.Online, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
+                   .Returns(async (Guid sessionId, TaskAgentStatus status, string runnerVersion, string os, string architecture, bool disableUpdate, CancellationToken cancellationToken) =>
+                   {
+                       counter++;
+                       await Task.Yield();
+                       if (counter == 2)
+                       {
+                           throw new NotSupportedException("Something wrong.");
+                       }
+
+                       return brokerMessageQueue.Dequeue();
+                   });
+
+                TaskAgentMessage message1 = await listener.GetNextMessageAsync(tokenSource.Token);
+                TaskAgentMessage message2 = await listener.GetNextMessageAsync(tokenSource.Token);
+                TaskAgentMessage message3 = await listener.GetNextMessageAsync(tokenSource.Token);
+                Assert.Equal(brokerMessages[0], message1);
+                Assert.Equal(brokerMessages[1], message2);
+                Assert.Equal(brokerMessages[4], message3);
+
+                //Assert
+                _runnerServer
+                    .Verify(x => x.GetAgentMessageAsync(
+                        _settings.PoolId, expectedSession.SessionId, It.IsAny<long?>(), TaskAgentStatus.Online, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()), Times.Exactly(brokerMessages.Length + 1));
+
+                _brokerServer
+                    .Verify(x => x.GetRunnerMessageAsync(
+                    expectedSession.SessionId, TaskAgentStatus.Online, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()), Times.Exactly(brokerMessages.Length + 1));
+
+                _credMgr
+                    .Verify(x => x.LoadCredentials(true), Times.Exactly(brokerMessages.Length + 1));
+
+                _brokerServer
+                    .Verify(x => x.UpdateConnectionIfNeeded(brokerMigrationMesage.BrokerBaseUrl, It.IsAny<VssCredentials>()), Times.Exactly(brokerMessages.Length + 1));
+
+                _brokerServer
+                    .Verify(x => x.ForceRefreshConnection(It.IsAny<VssCredentials>()), Times.Once());
+
+                Assert.False(tc.AllowAuthMigration);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async Task GetNextMessageWithBrokerMigration_EnableAuthMigration()
+        {
+            using (TestHostContext tc = CreateTestContext())
+            using (var tokenSource = new CancellationTokenSource())
+            {
+                Tracing trace = tc.GetTrace();
+
+                // Arrange.
+                var expectedSession = new TaskAgentSession();
+                PropertyInfo sessionIdProperty = expectedSession.GetType().GetProperty("SessionId", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
+                Assert.NotNull(sessionIdProperty);
+                sessionIdProperty.SetValue(expectedSession, Guid.NewGuid());
+
+                _runnerServer
+                    .Setup(x => x.CreateAgentSessionAsync(
+                        _settings.PoolId,
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedSession));
+
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
+                _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
+                _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
+
+                // Act.
+                MessageListener listener = new();
+                listener.Initialize(tc);
+
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
+                Assert.Equal(CreateSessionResult.Success, result);
+
+                var brokerMigrationMesage = new BrokerMigrationMessage(new Uri("https://actions.broker.com"));
+
+                var arMessages = new TaskAgentMessage[]
+                {
+                        new TaskAgentMessage
+                        {
+                            Body = JsonUtility.ToString(brokerMigrationMesage),
+                            MessageType = BrokerMigrationMessage.MessageType
+                        },
+                };
+
+                var brokerMessages = new TaskAgentMessage[]
+                {
+                        new TaskAgentMessage
+                        {
+                            Body = "somebody1",
+                            MessageId = 4234,
+                            MessageType = JobRequestMessageTypes.PipelineAgentJobRequest
+                        },
+                        new TaskAgentMessage
+                        {
+                            Body = "somebody2",
+                            MessageId = 4235,
+                            MessageType = JobCancelMessage.MessageType
+                        },
+                        null,  //should be skipped by GetNextMessageAsync implementation
+                        null,
+                        new TaskAgentMessage
+                        {
+                            Body = "somebody3",
+                            MessageId = 4236,
+                            MessageType = JobRequestMessageTypes.PipelineAgentJobRequest
+                        }
+                };
+                var brokerMessageQueue = new Queue<TaskAgentMessage>(brokerMessages);
+
+                _runnerServer
+                    .Setup(x => x.GetAgentMessageAsync(
+                        _settings.PoolId, expectedSession.SessionId, It.IsAny<long?>(), TaskAgentStatus.Online, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
+                    .Returns(async (Int32 poolId, Guid sessionId, Int64? lastMessageId, TaskAgentStatus status, string runnerVersion, string os, string architecture, bool disableUpdate, CancellationToken cancellationToken) =>
+                    {
+                        await Task.Yield();
+                        return arMessages[0]; // always send migration message
+                    });
+
+                _brokerServer
+                   .Setup(x => x.GetRunnerMessageAsync(
+                       expectedSession.SessionId, TaskAgentStatus.Online, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
+                   .Returns(async (Guid sessionId, TaskAgentStatus status, string runnerVersion, string os, string architecture, bool disableUpdate, CancellationToken cancellationToken) =>
+                   {
+                       await Task.Yield();
+                       if (!tc.AllowAuthMigration)
+                       {
+                           tc.EnableAuthMigration("L0Test");
+                       }
+
+                       return brokerMessageQueue.Dequeue();
+                   });
+
+                TaskAgentMessage message1 = await listener.GetNextMessageAsync(tokenSource.Token);
+                TaskAgentMessage message2 = await listener.GetNextMessageAsync(tokenSource.Token);
+                TaskAgentMessage message3 = await listener.GetNextMessageAsync(tokenSource.Token);
+                Assert.Equal(brokerMessages[0], message1);
+                Assert.Equal(brokerMessages[1], message2);
+                Assert.Equal(brokerMessages[4], message3);
+
+                //Assert
+                _runnerServer
+                    .Verify(x => x.GetAgentMessageAsync(
+                        _settings.PoolId, expectedSession.SessionId, It.IsAny<long?>(), TaskAgentStatus.Online, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()), Times.Exactly(brokerMessages.Length));
+
+                _brokerServer
+                    .Verify(x => x.GetRunnerMessageAsync(
+                    expectedSession.SessionId, TaskAgentStatus.Online, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()), Times.Exactly(brokerMessages.Length));
+
+                _credMgr
+                    .Verify(x => x.LoadCredentials(true), Times.Exactly(brokerMessages.Length));
+
+                _brokerServer
+                    .Verify(x => x.UpdateConnectionIfNeeded(brokerMigrationMesage.BrokerBaseUrl, It.IsAny<VssCredentials>()), Times.Exactly(brokerMessages.Length));
+
+                _brokerServer
+                    .Verify(x => x.ForceRefreshConnection(It.IsAny<VssCredentials>()), Times.Once());
+
+                Assert.True(tc.AllowAuthMigration);
+            }
+        }
     }
 }