fix release.yml break by upgrading actions/github-script part 2

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.devcontainer/devcontainer.json

@@ -4,10 +4,13 @@
   "features": {
     "ghcr.io/devcontainers/features/docker-in-docker:1": {},
     "ghcr.io/devcontainers/features/dotnet": {
-      "version": "6.0.418"
+      "version": "8.0.407"
     },
     "ghcr.io/devcontainers/features/node:1": {
-      "version": "16"
+      "version": "20"
+    },
+    "ghcr.io/devcontainers/features/sshd:1": {
+      "version": "latest"
     }
   },
   "customizations": {

.github/ISSUE_TEMPLATE/config.yml

@@ -7,7 +7,7 @@ contact_links:
     url: https://github.community/c/code-to-cloud/52
     about: If you have questions about GitHub Actions or need support writing workflows, please ask in the GitHub Community Support forum.
   - name: ✅ Feedback and suggestions for GitHub Actions
-    url: https://github.com/github/feedback/discussions/categories/actions-and-packages-feedback
+    url: https://github.com/github/feedback/discussions/categories/actions
     about: If you have feedback or suggestions about GitHub Actions, please open a discussion (or add to an existing one) in the GitHub Actions Feedback.  GitHub Actions Product Managers and Engineers monitor the feedback forum.
   - name: ‼️ GitHub Security Bug Bounty
     url: https://bounty.github.com/

.github/dependabot.yml

@@ -5,6 +5,11 @@ updates:
   schedule:
     interval: "daily" 
   target-branch: "main" 
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "daily" 
+  target-branch: "main" 
 - package-ecosystem: "nuget"
   directory: "/src"
   schedule:

.github/workflows/build.yml

@@ -50,7 +50,7 @@ jobs:
 
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Build runner layout
     - name: Build & Layout Release
@@ -69,13 +69,13 @@ jobs:
     - name: Package Release
       if: github.event_name != 'pull_request'
       run: |
-        ${{ matrix.devScript }} package Release
+        ${{ matrix.devScript }} package Release ${{ matrix.runtime }}
       working-directory: src
 
     # Upload runner package tar.gz/zip as artifact
     - name: Publish Artifact
       if: github.event_name != 'pull_request'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: runner-package-${{ matrix.runtime }}
         path: |

.github/workflows/close-bugs-bot.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v8
+      - uses: actions/stale@v9
         with:
           close-issue-message: "This issue does not seem to be a problem with the runner application, it concerns the GitHub actions platform more generally. Could you please post your feedback on the [GitHub Community Support Forum](https://github.com/orgs/community/discussions/categories/actions) which is actively monitored. Using the forum ensures that we route your problem to the correct team. 😃"
           exempt-issue-labels: "keep"

.github/workflows/close-features-bot.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v8
+      - uses: actions/stale@v9
         with:
           close-issue-message: "Thank you for your interest in the runner application and taking the time to provide your valuable feedback. We kindly ask you to redirect this feedback to the [GitHub Community Support Forum](https://github.com/orgs/community/discussions/categories/actions-and-packages) which our team actively monitors and would be a better place to start a discussion for new feature requests in GitHub Actions. For more information on this policy please [read our contribution guidelines](https://github.com/actions/runner#contribute). 😃"
           exempt-issue-labels: "keep"

.github/workflows/codeql.yml

@@ -23,11 +23,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       # Override language selection by uncommenting this and choosing your languages
       # with:
       #   languages: go, javascript, csharp, python, cpp, java
@@ -38,4 +38,4 @@ jobs:
       working-directory: src
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/dotnet-upgrade.yml

@@ -15,7 +15,7 @@ jobs:
       DOTNET_CURRENT_MAJOR_MINOR_VERSION: ${{ steps.fetch_current_version.outputs.DOTNET_CURRENT_MAJOR_MINOR_VERSION }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Get current major minor version
         id: fetch_current_version
         shell: bash
@@ -51,7 +51,7 @@ jobs:
         run: echo "::error links::feature/dotnet-sdk-upgrade${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }} https://github.com/actions/runner/tree/feature/dotnet-sdk-upgrade${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}::Branch feature/dotnetsdk-upgrade/${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }} already exists. Please take a look and delete that branch if you wish to recreate"
       - name: Create a warning annotation if no need to update
         if: ${{ steps.fetch_latest_version.outputs.SHOULD_UPDATE == 0 && steps.fetch_latest_version.outputs.BRANCH_EXISTS == 0 }}
-        run: echo "::warning ::Latest DotNet SDK patch is ${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}, and we are on ${{ steps.fetch_latest_version.outputs.DOTNET_CURRENT_MAJOR_MINOR_PATCH_VERSION }}. No need to update"
+        run: echo "::warning ::Latest DotNet SDK patch is ${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}, and we are on ${{ steps.fetch_current_version.outputs.DOTNET_CURRENT_MAJOR_MINOR_PATCH_VERSION }}. No need to update"
       - name: Update patch version
         if: ${{ steps.fetch_latest_version.outputs.SHOULD_UPDATE == 1 && steps.fetch_latest_version.outputs.BRANCH_EXISTS == 0 }}
         shell: bash
@@ -89,7 +89,7 @@ jobs:
     if: ${{ needs.dotnet-update.outputs.SHOULD_UPDATE == 1 && needs.dotnet-update.outputs.BRANCH_EXISTS == 0 }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: feature/dotnetsdk-upgrade/${{ needs.dotnet-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}
     - name: Create Pull Request

.github/workflows/lint.yml

@@ -1,24 +0,0 @@
-name: Lint
-
-on:
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-    name: Lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          # Ensure full list of changed files within `super-linter`
-          fetch-depth: 0
-      - name: Run linters
-        uses: github/super-linter@v4
-        env:
-          DEFAULT_BRANCH: ${{ github.base_ref }}
-          EDITORCONFIG_FILE_NAME: .editorconfig
-          LINTER_RULES_PATH: /src/
-          VALIDATE_ALL_CODEBASE: false
-          VALIDATE_CSHARP: true

.github/workflows/publish-image.yml

@@ -1,68 +0,0 @@
-name: Publish Runner Image
-
-on:
-  workflow_dispatch:
-    inputs:
-      runnerVersion:
-        type: string
-        description: Version of the runner being installed
-
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository_owner }}/actions-runner
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Compute image version
-        id: image
-        uses: actions/github-script@v6
-        with:
-          script: |
-            const fs = require('fs');
-            const inputRunnerVersion = "${{ github.event.inputs.runnerVersion }}"
-            if (inputRunnerVersion) {
-              console.log(`Using input runner version ${inputRunnerVersion}`)
-              core.setOutput('version', inputRunnerVersion);
-              return
-            }
-            const runnerVersion = fs.readFileSync('${{ github.workspace }}/src/runnerversion', 'utf8').replace(/\n$/g, '')
-            console.log(`Using runner version ${runnerVersion}`)
-            core.setOutput('version', runnerVersion);
-
-      - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push Docker image
-        id: build-and-push
-        uses: docker/build-push-action@v3
-        with:
-          context: ./images
-          platforms: |
-            linux/amd64
-            linux/arm64
-          tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.image.outputs.version }}
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-          build-args: |
-            RUNNER_VERSION=${{ steps.image.outputs.version }}
-          push: true
-          labels: |
-            org.opencontainers.image.source=${{github.server_url}}/${{github.repository}}
-            org.opencontainers.image.description=https://github.com/actions/runner/releases/tag/v${{ steps.image.outputs.version }}
-            org.opencontainers.image.licenses=MIT

.github/workflows/release.yml

@@ -11,16 +11,15 @@ jobs:
     if: startsWith(github.ref, 'refs/heads/releases/') || github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Make sure ./releaseVersion match ./src/runnerversion
     # Query GitHub release ensure version is not used
     - name: Check version
-      uses: actions/github-script@0.3.0
+      uses: actions/github-script@v7.0.1
       with:
         github-token: ${{secrets.GITHUB_TOKEN}}
         script: |
-          const core = require('@actions/core')
           const fs = require('fs');
           const runnerVersion = fs.readFileSync('${{ github.workspace }}/src/runnerversion', 'utf8').replace(/\n$/g, '')
           const releaseVersion = fs.readFileSync('${{ github.workspace }}/releaseVersion', 'utf8').replace(/\n$/g, '')
@@ -30,7 +29,7 @@ jobs:
             return
           }
           try {
-            const release = await github.repos.getReleaseByTag({
+            const release = await github.rest.repos.getReleaseByTag({
               owner: '${{ github.event.repository.owner.name }}',
               repo: '${{ github.event.repository.name }}',
               tag: 'v' + runnerVersion
@@ -87,7 +86,7 @@ jobs:
 
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Build runner layout
     - name: Build & Layout Release
@@ -117,12 +116,11 @@ jobs:
       working-directory: _package
 
     # Upload runner package tar.gz/zip as artifact.
-    # Since each package name is unique, so we don't need to put ${{matrix}} info into artifact name
     - name: Publish Artifact
       if: github.event_name != 'pull_request'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
-        name: runner-packages
+        name: runner-packages-${{ matrix.runtime }}
         path: |
           _package
 
@@ -131,23 +129,52 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # Download runner package tar.gz/zip produced by 'build' job
-    - name: Download Artifact
-      uses: actions/download-artifact@v1
+    - name: Download Artifact (win-x64)
+      uses: actions/download-artifact@v4
       with:
-        name: runner-packages
+        name: runner-packages-win-x64
+        path: ./
+    - name: Download Artifact (win-arm64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-win-arm64
+        path: ./
+    - name: Download Artifact (osx-x64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-osx-x64
+        path: ./
+    - name: Download Artifact (osx-arm64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-osx-arm64
+        path: ./
+    - name: Download Artifact (linux-x64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-linux-x64
+        path: ./
+    - name: Download Artifact (linux-arm)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-linux-arm
+        path: ./
+    - name: Download Artifact (linux-arm64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-linux-arm64
         path: ./
 
     # Create ReleaseNote file
     - name: Create ReleaseNote
       id: releaseNote
-      uses: actions/github-script@0.3.0
+      uses: actions/github-script@v7.0.1
       with:
         github-token: ${{secrets.GITHUB_TOKEN}}
         script: |
-          const core = require('@actions/core')
           const fs = require('fs');
           const runnerVersion = fs.readFileSync('${{ github.workspace }}/src/runnerversion', 'utf8').replace(/\n$/g, '')
           var releaseNote = fs.readFileSync('${{ github.workspace }}/releaseNote.md', 'utf8').replace(/<RUNNER_VERSION>/g, runnerVersion)
@@ -163,7 +190,6 @@ jobs:
           core.setOutput('note', releaseNote);
 
     - name: Validate Packages HASH
-      working-directory: _package
       run: |
         ls -l
         echo "${{needs.build.outputs.win-x64-sha}}  actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip" | shasum -a 256 -c
@@ -193,7 +219,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip
+        asset_path: ${{ github.workspace }}/actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip
         asset_name: actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip
         asset_content_type: application/octet-stream
 
@@ -203,7 +229,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}.zip
+        asset_path: ${{ github.workspace }}/actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}.zip
         asset_name: actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}.zip
         asset_content_type: application/octet-stream
 
@@ -213,7 +239,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
@@ -223,7 +249,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
@@ -233,7 +259,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
@@ -243,7 +269,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
@@ -253,7 +279,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
@@ -263,16 +289,18 @@ jobs:
     permissions:
       contents: read
       packages: write
+      id-token: write
+      attestations: write
     env:
       REGISTRY: ghcr.io
       IMAGE_NAME: ${{ github.repository_owner }}/actions-runner
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Compute image version
         id: image
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7.0.1
         with:
           script: |
             const fs = require('fs');
@@ -281,10 +309,10 @@ jobs:
             core.setOutput('version', runnerVersion);
 
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -292,7 +320,7 @@ jobs:
 
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: ./images
           platforms: |
@@ -308,3 +336,10 @@ jobs:
             org.opencontainers.image.source=${{github.server_url}}/${{github.repository}}
             org.opencontainers.image.description=https://github.com/actions/runner/releases/tag/v${{ steps.image.outputs.version }}
             org.opencontainers.image.licenses=MIT
+
+      - name: Generate attestation
+        uses: actions/attest-build-provenance@v2
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          subject-digest: ${{ steps.build-and-push.outputs.digest }}
+          push-to-registry: true

.github/workflows/stale-bot.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v8
+      - uses: actions/stale@v9
         with:
           stale-issue-message: "This issue is stale because it has been open 365 days with no activity. Remove stale label or comment or this will be closed in 15 days."
           close-issue-message: "This issue was closed because it has been stalled for 15 days with no activity."

.gitignore

@@ -26,4 +26,5 @@ _dotnetsdk
 TestResults
 TestLogs
 .DS_Store
+.mono
 **/*.DotSettings.user

docs/adrs/1751-runner-job-hooks.md

@@ -23,7 +23,7 @@ This feature is mainly intended for self hosted runner administrators.
   - `ACTIONS_RUNNER_HOOK_JOB_STARTED`
   - `ACTIONS_RUNNER_HOOK_JOB_COMPLETED`
 
-You can set these variables to the **absolute** path of a a `.sh` or `.ps1` file.
+You can set these variables to the **absolute** path of a `.sh` or `.ps1` file.
 
 We will execute `pwsh` (fallback to `powershell`) or `bash` (fallback to `sh`) as appropriate.
 - `.sh` files will execute with the args `-e {pathtofile}`

docs/checks/nodejs.md

@@ -4,9 +4,9 @@
 
 Make sure the built-in node.js has access to GitHub.com or GitHub Enterprise Server.
 
-The runner carries its own copy of node.js executable under `<runner_root>/externals/node16/`.
+The runner carries its own copy of node.js executable under `<runner_root>/externals/node20/`.
 
-All javascript base Actions will get executed by the built-in `node` at `<runner_root>/externals/node16/`.
+All javascript base Actions will get executed by the built-in `node` at `<runner_root>/externals/node20/`.
 
 > Not the `node` from `$PATH`
 

docs/contribute.md

@@ -1,6 +1,6 @@
 # Contributions
 
-We welcome contributions in the form of issues and pull requests. We view the contributions and the process as the same for github and external contributors.Please note the runner typically requires changes across the entire system and we aim for issues in the runner to be entirely self contained and fixable here. Therefore, we will primarily handle bug issues opened in this repo and we kindly request you to create all feature and enhancement requests on the [GitHub Feedback](https://github.com/community/community/discussions/categories/actions-and-packages) page. 
+We welcome contributions in the form of issues and pull requests. We view the contributions and the process as the same for github and external contributors. Please note the runner typically requires changes across the entire system and we aim for issues in the runner to be entirely self contained and fixable here. Therefore, we will primarily handle bug issues opened in this repo and we kindly request you to create all feature and enhancement requests on the [GitHub Feedback](https://github.com/community/community/discussions/categories/actions-and-packages) page. 
 
 > IMPORTANT: Building your own runner is critical for the dev inner loop process when contributing changes.  However, only runners built and distributed by GitHub (releases) are supported in production.  Be aware that workflows and orchestrations run service side with the runner being a remote process to run steps.  For that reason, the service can pull the runner forward so customizations can be lost.
 

docs/start/envlinux.md

@@ -4,16 +4,7 @@
 
 ## Supported Distributions and Versions
 
-x64
-  - Red Hat Enterprise Linux 7+
-  - CentOS 7+
-  - Oracle Linux 7+
-  - Fedora 29+
-  - Debian 9+
-  - Ubuntu 16.04+
-  - Linux Mint 18+
-  - openSUSE 15+
-  - SUSE Enterprise Linux (SLES) 12 SP2+
+Please see "[Supported architectures and operating systems for self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#linux)."
 
 ## Install .Net Core 3.x Linux Dependencies
 

docs/start/envosx.md

@@ -4,7 +4,6 @@
 
 ## Supported Versions
 
-  - macOS High Sierra (10.13) and later versions
-  - x64 and arm64 (Apple Silicon)
+Please see "[Supported architectures and operating systems for self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#macos)."
 
 ## [More .Net Core Prerequisites Information](https://docs.microsoft.com/en-us/dotnet/core/macos-prerequisites?tabs=netcore30)

docs/start/envwin.md

@@ -2,11 +2,6 @@
 
 ## Supported Versions
 
- - Windows 7 64-bit
- - Windows 8.1 64-bit
- - Windows 10 64-bit
- - Windows Server 2012 R2 64-bit
- - Windows Server 2016 64-bit
- - Windows Server 2019 64-bit
+Please see "[Supported architectures and operating systems for self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#windows)."
 
 ## [More .NET Core Prerequisites Information](https://docs.microsoft.com/en-us/dotnet/core/windows-prerequisites?tabs=netcore30)

images/Dockerfile

@@ -1,12 +1,12 @@
 # Source: https://github.com/dotnet/dotnet-docker
-FROM mcr.microsoft.com/dotnet/runtime-deps:6.0-jammy as build
+FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-jammy AS build
 
 ARG TARGETOS
 ARG TARGETARCH
 ARG RUNNER_VERSION
-ARG RUNNER_CONTAINER_HOOKS_VERSION=0.5.0
-ARG DOCKER_VERSION=24.0.6
-ARG BUILDX_VERSION=0.11.2
+ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.1
+ARG DOCKER_VERSION=28.0.1
+ARG BUILDX_VERSION=0.21.2
 
 RUN apt update -y && apt install curl unzip -y
 
@@ -32,17 +32,22 @@ RUN export RUNNER_ARCH=${TARGETARCH} \
         "https://github.com/docker/buildx/releases/download/v${BUILDX_VERSION}/buildx-v${BUILDX_VERSION}.linux-${TARGETARCH}" \
     && chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx
 
-FROM mcr.microsoft.com/dotnet/runtime-deps:6.0-jammy
+FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-jammy
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV RUNNER_MANUALLY_TRAP_SIG=1
 ENV ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT=1
 ENV ImageOS=ubuntu22
 
-RUN apt-get update -y \
-    && apt-get install -y --no-install-recommends \
-    sudo \
-    lsb-release \
+# 'gpg-agent' and 'software-properties-common' are needed for the 'add-apt-repository' command that follows
+RUN apt update -y \
+    && apt install -y --no-install-recommends sudo lsb-release gpg-agent software-properties-common curl jq unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+# Configure git-core/ppa based on guidance here:  https://git-scm.com/download/linux
+RUN add-apt-repository ppa:git-core/ppa \
+    && apt update -y \
+    && apt install -y git \
     && rm -rf /var/lib/apt/lists/*
 
 RUN adduser --disabled-password --gecos "" --uid 1001 runner \

releaseNote.md

@@ -1,36 +1,48 @@
 ## What's Changed
-* Fix `buildx` installation by @ajschmidt8 in https://github.com/actions/runner/pull/2952
-* Create close-features and close-bugs bot for runner issues by @ruvceskistefan in https://github.com/actions/runner/pull/2909
-* Send disableUpdate as query parameter  by @luketomlinson in https://github.com/actions/runner/pull/2970
-* Handle SelfUpdate Flow when Package is provided in Message  by @luketomlinson in https://github.com/actions/runner/pull/2926
-* Bump container hook version to 0.5.0 in runner image by @nikola-jokic in https://github.com/actions/runner/pull/3003
-* Set `ImageOS` environment variable in runner images by @int128 in https://github.com/actions/runner/pull/2878
-* Mark job as failed on worker crash. by @TingluoHuang in https://github.com/actions/runner/pull/3006
-* Include whether http proxy configured as part of UserAgent. by @TingluoHuang in https://github.com/actions/runner/pull/3009
-* Add codeload to the list of service we check during '--check'. by @TingluoHuang in https://github.com/actions/runner/pull/3011
-* close reason update by @ruvceskistefan in https://github.com/actions/runner/pull/3027
-* Update envlinux.md by @adjn in https://github.com/actions/runner/pull/3040
-* Extend `--check` to check Results-Receiver service. by @TingluoHuang in https://github.com/actions/runner/pull/3078
-* Use Azure SDK to upload files to Azure Blob by @yacaovsnc in https://github.com/actions/runner/pull/3033
-* Remove code in runner for handling trimmed packages. by @TingluoHuang in https://github.com/actions/runner/pull/3074
-* Update dotnet sdk to latest version @6.0.418 by @github-actions in https://github.com/actions/runner/pull/3085
-* Patch Curl to no longer use -k by @thboop in https://github.com/actions/runner/pull/3091
+* Bump docker/login-action from 2 to 3 by @dependabot in https://github.com/actions/runner/pull/3673
+* Bump actions/stale from 8 to 9 by @dependabot in https://github.com/actions/runner/pull/3554
+* Bump docker/build-push-action from 3 to 6 by @dependabot in https://github.com/actions/runner/pull/3674
+* update node version from 20.18.0 -> 20.18.2 by @aiqiaoy in https://github.com/actions/runner/pull/3682
+* Pass BillingOwnerId through Acquire/Complete calls by @luketomlinson in https://github.com/actions/runner/pull/3689
+* Do not retry CompleteJobAsync for known non-retryable errors by @ericsciple in https://github.com/actions/runner/pull/3696
+* Update dotnet sdk to latest version @8.0.406 by @github-actions in https://github.com/actions/runner/pull/3712
+* Update Dockerfile with new docker and buildx versions by @thboop in https://github.com/actions/runner/pull/3680
+* chore: remove redundant words by @finaltrip in https://github.com/actions/runner/pull/3705
+* fix: actions feedback link is incorrect by @Yaminyam in https://github.com/actions/runner/pull/3165
+* Bump actions/github-script from 0.3.0 to 7.0.1 by @dependabot in https://github.com/actions/runner/pull/3557
+* Docker container provenance by @paveliak in https://github.com/actions/runner/pull/3736
+* Add request-id to http eventsource trace. by @TingluoHuang in https://github.com/actions/runner/pull/3740
+* Update Bocker and Buildx version to mitigate images scanners alerts by @Blizter in https://github.com/actions/runner/pull/3750
+* Fix typo, add invariant culture to timestamp for workflow log reporting by @GhadimiR in https://github.com/actions/runner/pull/3749
+* Create vssconnection to actions service when URL provided. by @TingluoHuang in https://github.com/actions/runner/pull/3751
+* Housekeeping: Update npm packages and node version by @thboop in https://github.com/actions/runner/pull/3752
+* Improve the out-of-date warning message. by @tecimovic in https://github.com/actions/runner/pull/3595
+* Update dotnet sdk to latest version @8.0.407 by @github-actions in https://github.com/actions/runner/pull/3753
+* Exit hosted runner cleanly during deprovisioning. by @TingluoHuang in https://github.com/actions/runner/pull/3755
+* Send annotation title to run-service. by @TingluoHuang in https://github.com/actions/runner/pull/3757
+* Allow server enforce runner settings. by @TingluoHuang in https://github.com/actions/runner/pull/3758
+* Support refresh runner configs with pipelines service. by @TingluoHuang in https://github.com/actions/runner/pull/3706
 
 ## New Contributors
-* @int128 made their first contribution in https://github.com/actions/runner/pull/2878
-* @adjn made their first contribution in https://github.com/actions/runner/pull/3040
+* @finaltrip made their first contribution in https://github.com/actions/runner/pull/3705
+* @Yaminyam made their first contribution in https://github.com/actions/runner/pull/3165
+* @Blizter made their first contribution in https://github.com/actions/runner/pull/3750
+* @GhadimiR made their first contribution in https://github.com/actions/runner/pull/3749
+* @tecimovic made their first contribution in https://github.com/actions/runner/pull/3595
 
-**Full Changelog**: https://github.com/actions/runner/compare/v2.311.0...v2.312.0
+**Full Changelog**: https://github.com/actions/runner/compare/v2.322.0...v2.323.0
 
-_Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. 
-To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. 
+_Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet.
+To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository.
 See https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/adding-self-hosted-runners_
 
 ## Windows x64
+
 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.
 
 The following snipped needs to be run on `powershell`:
-``` powershell
+
+```powershell
 # Create a folder under the drive root
 mkdir \actions-runner ; cd \actions-runner
 # Download the latest runner package
@@ -40,13 +52,13 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem ;
 [System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD\actions-runner-win-x64-<RUNNER_VERSION>.zip", "$PWD")
 ```
 
-## [Pre-release] Windows arm64
-**Warning:** Windows arm64 runners are currently in preview status and use [unofficial versions of nodejs](https://unofficial-builds.nodejs.org/). They are not intended for production workflows.
+## Windows arm64
 
 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.
 
 The following snipped needs to be run on `powershell`:
-``` powershell
+
+```powershell
 # Create a folder under the drive root
 mkdir \actions-runner ; cd \actions-runner
 # Download the latest runner package
@@ -58,7 +70,7 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem ;
 
 ## OSX x64
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -69,7 +81,7 @@ tar xzf ./actions-runner-osx-x64-<RUNNER_VERSION>.tar.gz
 
 ## OSX arm64 (Apple silicon)
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -80,7 +92,7 @@ tar xzf ./actions-runner-osx-arm64-<RUNNER_VERSION>.tar.gz
 
 ## Linux x64
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -91,7 +103,7 @@ tar xzf ./actions-runner-linux-x64-<RUNNER_VERSION>.tar.gz
 
 ## Linux arm64
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -102,7 +114,7 @@ tar xzf ./actions-runner-linux-arm64-<RUNNER_VERSION>.tar.gz
 
 ## Linux arm
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -112,6 +124,7 @@ tar xzf ./actions-runner-linux-arm-<RUNNER_VERSION>.tar.gz
 ```
 
 ## Using your self hosted runner
+
 For additional details about configuring, running, or shutting down the runner please check out our [product docs.](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/adding-self-hosted-runners)
 
 ## SHA-256 Checksums

releaseVersion

@@ -1 +1 @@
-<Update to ./src/runnerversion when creating release>
+2.323.0

src/Directory.Build.props

@@ -57,4 +57,13 @@
   <PropertyGroup>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
   </PropertyGroup>
+
+  <PropertyGroup>
+    <!-- Enable NuGet package auditing -->
+    <NuGetAudit>true</NuGetAudit>
+    <!-- Audit direct and transitive packages -->
+    <NuGetAuditMode>all</NuGetAuditMode>
+    <!-- Report low, moderate, high and critical advisories -->
+    <NuGetAuditLevel>moderate</NuGetAuditLevel>
+  </PropertyGroup>
 </Project>

src/Misc/contentHash/dotnetRuntime/linux-arm

@@ -1 +0,0 @@
-54d95a44d118dba852395991224a6b9c1abe916858c87138656f80c619e85331

src/Misc/contentHash/dotnetRuntime/linux-arm64

@@ -1 +0,0 @@
-68015af17f06a824fa478e62ae7393766ce627fd5599ab916432a14656a19a52

src/Misc/contentHash/dotnetRuntime/linux-x64

@@ -1 +0,0 @@
-a2628119ca419cb54e279103ffae7986cdbd0814d57c73ff0dc74c38be08b9ae

src/Misc/contentHash/dotnetRuntime/osx-arm64

@@ -1 +0,0 @@
-de71ca09ead807e1a2ce9df0a5b23eb7690cb71fff51169a77e4c3992be53dda

src/Misc/contentHash/dotnetRuntime/osx-x64

@@ -1 +0,0 @@
-d009e05e6b26d614d65be736a15d1bd151932121c16a9ff1b986deadecc982b9

src/Misc/contentHash/dotnetRuntime/win-arm64

@@ -1 +0,0 @@
-f730db39c2305800b4653795360ba9c10c68f384a46b85d808f1f9f0ed3c42e4

src/Misc/contentHash/dotnetRuntime/win-x64

@@ -1 +0,0 @@
-a35b5722375490e9473cdcccb5e18b41eba3dbf4344fe31abc9821e21f18ea5a

src/Misc/contentHash/externals/linux-arm

@@ -1 +0,0 @@
-4bf3e1af0d482af1b2eaf9f08250248a8c1aea8ec20a3c5be116d58cdd930009

src/Misc/contentHash/externals/linux-arm64

@@ -1 +0,0 @@
-ec1719a8cb4d8687328aa64f4aa7c4e3498a715d8939117874782e3e6e63a14b

src/Misc/contentHash/externals/linux-x64

@@ -1 +0,0 @@
-50538de29f173bb73f708c4ed2c8328a62b8795829b97b2a6cb57197e2305287

src/Misc/contentHash/externals/osx-arm64

@@ -1 +0,0 @@
-a0a96cbb7593643b69e669bf14d7b29b7f27800b3a00bb3305aebe041456c701

src/Misc/contentHash/externals/osx-x64

@@ -1 +0,0 @@
-6255b22692779467047ecebd60ad46984866d75cdfe10421d593a7b51d620b09

src/Misc/contentHash/externals/win-arm64

@@ -1 +0,0 @@
-6ff1abd055dc35bfbf06f75c2f08908f660346f66ad1d8f81c910068e9ba029d

src/Misc/contentHash/externals/win-x64

@@ -1 +0,0 @@
-433a6d748742d12abd20dc2a79b62ac3d9718ae47ef26f8e84dc8c180eea3659

src/Misc/expressionFunc/hashFiles/package.json

@@ -11,7 +11,6 @@
     "pack": "ncc build -o ../../layoutbin/hashFiles",
     "all": "npm run format && npm run lint && npm run build && npm run pack",
     "prepare": "cd ../../../../ && husky install"
-
   },
   "repository": {
     "type": "git",
@@ -43,9 +42,9 @@
     "eslint": "^8.47.0",
     "eslint-plugin-github": "^4.10.0",
     "eslint-plugin-prettier": "^5.0.0",
-    "prettier": "^3.0.3",
-    "typescript": "^5.2.2",
     "husky": "^8.0.3",
-    "lint-staged": "^14.0.0"
+    "lint-staged": "^15.5.0",
+    "prettier": "^3.0.3",
+    "typescript": "^5.2.2"
   }
-}
+}

src/Misc/externals.sh

@@ -3,12 +3,10 @@ PACKAGERUNTIME=$1
 PRECACHE=$2
 
 NODE_URL=https://nodejs.org/dist
-UNOFFICIAL_NODE_URL=https://unofficial-builds.nodejs.org/download/release
 NODE_ALPINE_URL=https://github.com/actions/alpine_nodejs/releases/download
-NODE16_VERSION="16.20.2"
-NODE20_VERSION="20.8.1"
-# used only for win-arm64, remove node16 unofficial version when official version is available
-NODE16_UNOFFICIAL_VERSION="16.20.0"
+# When you update Node versions you must also create a new release of alpine_nodejs at that updated version.
+# Follow the instructions here: https://github.com/actions/alpine_nodejs?tab=readme-ov-file#getting-started
+NODE20_VERSION="20.19.0"
 
 get_abs_path() {
   # exploits the fact that pwd will print abs path when no args
@@ -139,8 +137,6 @@ function acquireExternalTool() {
 
 # Download the external tools only for Windows.
 if [[ "$PACKAGERUNTIME" == "win-x64" || "$PACKAGERUNTIME" == "win-x86" ]]; then
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/$PACKAGERUNTIME/node.exe" node16/bin
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/$PACKAGERUNTIME/node.lib" node16/bin
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/$PACKAGERUNTIME/node.exe" node20/bin
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/$PACKAGERUNTIME/node.lib" node20/bin
     if [[ "$PRECACHE" != "" ]]; then
@@ -151,8 +147,6 @@ fi
 # Download the external tools only for Windows.
 if [[ "$PACKAGERUNTIME" == "win-arm64" ]]; then
     # todo: replace these with official release when available
-    acquireExternalTool "$UNOFFICIAL_NODE_URL/v${NODE16_UNOFFICIAL_VERSION}/$PACKAGERUNTIME/node.exe" node16/bin
-    acquireExternalTool "$UNOFFICIAL_NODE_URL/v${NODE16_UNOFFICIAL_VERSION}/$PACKAGERUNTIME/node.lib" node16/bin
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/$PACKAGERUNTIME/node.exe" node20/bin
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/$PACKAGERUNTIME/node.lib" node20/bin
     if [[ "$PRECACHE" != "" ]]; then
@@ -162,30 +156,24 @@ fi
 
 # Download the external tools only for OSX.
 if [[ "$PACKAGERUNTIME" == "osx-x64" ]]; then
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-darwin-x64.tar.gz" node16 fix_nested_dir
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-darwin-x64.tar.gz" node20 fix_nested_dir
 fi
 
 if [[ "$PACKAGERUNTIME" == "osx-arm64" ]]; then
     # node.js v12 doesn't support macOS on arm64.
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-darwin-arm64.tar.gz" node16 fix_nested_dir
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-darwin-arm64.tar.gz" node20 fix_nested_dir
 fi
 
 # Download the external tools for Linux PACKAGERUNTIMEs.
 if [[ "$PACKAGERUNTIME" == "linux-x64" ]]; then
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-linux-x64.tar.gz" node16 fix_nested_dir
-    acquireExternalTool "$NODE_ALPINE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-alpine-x64.tar.gz" node16_alpine
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-linux-x64.tar.gz" node20 fix_nested_dir
     acquireExternalTool "$NODE_ALPINE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-alpine-x64.tar.gz" node20_alpine
 fi
 
 if [[ "$PACKAGERUNTIME" == "linux-arm64" ]]; then
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-linux-arm64.tar.gz" node16 fix_nested_dir
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-linux-arm64.tar.gz" node20 fix_nested_dir
 fi
 
 if [[ "$PACKAGERUNTIME" == "linux-arm" ]]; then
-    acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-linux-armv7l.tar.gz" node16 fix_nested_dir
     acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-linux-armv7l.tar.gz" node20 fix_nested_dir
 fi

src/Misc/layoutbin/RunnerService.js

@@ -114,6 +114,11 @@ var runService = function () {
             );
             stopping = true;
           }
+        } else if (code === 5) {
+          console.log(
+            "Runner listener exit with Session Conflict error, stop the service, no retry needed."
+          );
+          stopping = true;
         } else {
           var messagePrefix = "Runner listener exit with undefined return code";
           unknownFailureRetryCount++;

src/Misc/layoutbin/runsvc.sh

@@ -10,7 +10,7 @@ if [ -f ".path" ]; then
     echo ".path=${PATH}"
 fi
 
-nodever=${GITHUB_ACTIONS_RUNNER_FORCED_NODE_VERSION:-node16}
+nodever="node20"
 
 # insert anything to setup env when running as a service
 # run the host process which keep the listener alive

src/Misc/layoutbin/update.sh.template

@@ -135,12 +135,17 @@ if [[ "$currentplatform" == 'darwin'  && restartinteractiverunner -eq 0 ]]; then
     then
         # inspect the open file handles to find the node process
         # we can't actually inspect the process using ps because it uses relative paths and doesn't follow symlinks
-        nodever="node16"
+        nodever="node20"
         path=$(lsof -a -g "$procgroup" -F n | grep $nodever/bin/node | grep externals | tail -1 | cut -c2-)
-        if [[ $? -ne 0 || -z "$path" ]] # Fallback if RunnerService.js was started with node12
+        if [[ $? -ne 0 || -z "$path" ]] # Fallback if RunnerService.js was started with node16
         then
-            nodever="node12"
+            nodever="node16"
             path=$(lsof -a -g "$procgroup" -F n | grep $nodever/bin/node | grep externals | tail -1 | cut -c2-)
+            if [[ $? -ne 0 || -z "$path" ]] # Fallback if RunnerService.js was started with node12
+            then
+                nodever="node12"
+                path=$(lsof -a -g "$procgroup" -F n | grep $nodever/bin/node | grep externals | tail -1 | cut -c2-)
+            fi
         fi
         if [[ $? -eq 0 && -n "$path" ]]
         then
@@ -178,6 +183,19 @@ if [[ "$currentplatform" == 'darwin'  && restartinteractiverunner -eq 0 ]]; then
     fi
 fi
 
+# update runsvc.sh
+if [ -f "$rootfolder/runsvc.sh" ]
+then
+    date "+[%F %T-%4N] Update runsvc.sh" >> "$logfile" 2>&1
+    cat "$rootfolder/bin/runsvc.sh" > "$rootfolder/runsvc.sh"
+    if [ $? -ne 0 ]
+    then
+        date "+[%F %T-%4N] Can't update $rootfolder/runsvc.sh using $rootfolder/bin/runsvc.sh" >> "$logfile" 2>&1
+        mv -fv "$logfile" "$logfile.failed"
+        exit 1
+    fi
+fi
+
 date "+[%F %T-%4N] Update succeed" >> "$logfile"
 
 touch update.finished

src/Misc/layoutroot/run-helper.cmd.template

@@ -49,5 +49,10 @@ if %ERRORLEVEL% EQU 4 (
   exit /b 1
 )
 
+if %ERRORLEVEL% EQU 5 (
+  echo "Runner listener exit with Session Conflict error, stop the service, no retry needed."
+  exit /b 0
+)
+
 echo "Exiting after unknown error code: %ERRORLEVEL%"
 exit /b 0

src/Misc/layoutroot/run-helper.sh.template

@@ -70,6 +70,9 @@ elif [[ $returnCode == 4 ]]; then
         "$DIR"/safe_sleep.sh 1
     done
     exit 2
+elif [[ $returnCode == 5 ]]; then
+    echo "Runner listener exit with Session Conflict error, stop the service, no retry needed."
+    exit 0
 else
     echo "Exiting with unknown error code: ${returnCode}"
     exit 0

src/Misc/layoutroot/run.sh

@@ -38,7 +38,7 @@ runWithManualTrap() {
         cp -f "$DIR"/run-helper.sh.template "$DIR"/run-helper.sh
         "$DIR"/run-helper.sh $* &
         PID=$!
-        wait -f $PID
+        wait $PID
         returnCode=$?
         if [[ $returnCode -eq 2 ]]; then
             echo "Restarting runner..."
@@ -84,4 +84,4 @@ if [[ -z "$RUNNER_MANUALLY_TRAP_SIG" ]]; then
     run $*
 else
     runWithManualTrap $*
-fi
+fi

src/Runner.Common/ActionsRunServer.cs

@@ -20,12 +20,12 @@ namespace GitHub.Runner.Common
     {
         private bool _hasConnection;
         private VssConnection _connection;
-        private TaskAgentHttpClient _taskAgentClient;
+        private ActionsRunServerHttpClient _actionsRunServerClient;
 
         public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials)
         {
             _connection = await EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
-            _taskAgentClient = _connection.GetClient<TaskAgentHttpClient>();
+            _actionsRunServerClient = _connection.GetClient<ActionsRunServerHttpClient>();
             _hasConnection = true;
         }
 
@@ -42,7 +42,7 @@ namespace GitHub.Runner.Common
             CheckConnection();
             var jobMessage = RetryRequest<AgentJobRequestMessage>(async () =>
                                                     {
-                                                        return await _taskAgentClient.GetJobMessageAsync(id, cancellationToken);
+                                                        return await _actionsRunServerClient.GetJobMessageAsync(id, cancellationToken);
                                                     }, cancellationToken);
 
             return jobMessage;

src/Runner.Common/BrokerServer.cs

@@ -7,6 +7,7 @@ using GitHub.DistributedTask.Pipelines;
 using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
+using GitHub.Services.WebApi;
 using Sdk.RSWebApi.Contracts;
 using Sdk.WebApi.WebApi.RawClient;
 
@@ -17,7 +18,14 @@ namespace GitHub.Runner.Common
     {
         Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
 
-        Task<TaskAgentMessage> GetRunnerMessageAsync(CancellationToken token, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate);
+        Task<TaskAgentSession> CreateSessionAsync(TaskAgentSession session, CancellationToken cancellationToken);
+        Task DeleteSessionAsync(CancellationToken cancellationToken);
+
+        Task<TaskAgentMessage> GetRunnerMessageAsync(Guid? sessionId, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate, CancellationToken token);
+
+        Task UpdateConnectionIfNeeded(Uri serverUri, VssCredentials credentials);
+
+        Task ForceRefreshConnection(VssCredentials credentials);
     }
 
     public sealed class BrokerServer : RunnerService, IBrokerServer
@@ -44,13 +52,53 @@ namespace GitHub.Runner.Common
             }
         }
 
-        public Task<TaskAgentMessage> GetRunnerMessageAsync(CancellationToken cancellationToken, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate)
+        public async Task<TaskAgentSession> CreateSessionAsync(TaskAgentSession session, CancellationToken cancellationToken)
         {
             CheckConnection();
-            var jobMessage = RetryRequest<TaskAgentMessage>(
-                async () => await _brokerHttpClient.GetRunnerMessageAsync(version, status, os, architecture, disableUpdate, cancellationToken), cancellationToken);
+            var jobMessage = await _brokerHttpClient.CreateSessionAsync(session, cancellationToken);
 
             return jobMessage;
         }
+
+        public Task<TaskAgentMessage> GetRunnerMessageAsync(Guid? sessionId, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate, CancellationToken cancellationToken)
+        {
+            CheckConnection();
+            var brokerSession = RetryRequest<TaskAgentMessage>(
+                async () => await _brokerHttpClient.GetRunnerMessageAsync(sessionId, version, status, os, architecture, disableUpdate, cancellationToken), cancellationToken, shouldRetry: ShouldRetryException);
+
+
+            return brokerSession;
+        }
+
+        public async Task DeleteSessionAsync(CancellationToken cancellationToken)
+        {
+            CheckConnection();
+            await _brokerHttpClient.DeleteSessionAsync(cancellationToken);
+        }
+
+        public Task UpdateConnectionIfNeeded(Uri serverUri, VssCredentials credentials)
+        {
+            if (_brokerUri != serverUri || !_hasConnection)
+            {
+                return ConnectAsync(serverUri, credentials);
+            }
+
+            return Task.CompletedTask;
+        }
+
+        public Task ForceRefreshConnection(VssCredentials credentials)
+        {
+            return ConnectAsync(_brokerUri, credentials);
+        }
+
+        public bool ShouldRetryException(Exception ex)
+        {
+            if (ex is AccessDeniedException || ex is RunnerNotFoundException || ex is HostedRunnerDeprovisionedException)
+            {
+                return false;
+            }
+
+            return true;
+        }
     }
 }

src/Runner.Common/ConfigurationStore.cs

@@ -119,8 +119,11 @@ namespace GitHub.Runner.Common
         CredentialData GetCredentials();
         CredentialData GetMigratedCredentials();
         RunnerSettings GetSettings();
+        RunnerSettings GetMigratedSettings();
         void SaveCredential(CredentialData credential);
+        void SaveMigratedCredential(CredentialData credential);
         void SaveSettings(RunnerSettings settings);
+        void SaveMigratedSettings(RunnerSettings settings);
         void DeleteCredential();
         void DeleteMigratedCredential();
         void DeleteSettings();
@@ -130,6 +133,7 @@ namespace GitHub.Runner.Common
     {
         private string _binPath;
         private string _configFilePath;
+        private string _migratedConfigFilePath;
         private string _credFilePath;
         private string _migratedCredFilePath;
         private string _serviceConfigFilePath;
@@ -137,6 +141,7 @@ namespace GitHub.Runner.Common
         private CredentialData _creds;
         private CredentialData _migratedCreds;
         private RunnerSettings _settings;
+        private RunnerSettings _migratedSettings;
 
         public override void Initialize(IHostContext hostContext)
         {
@@ -154,6 +159,9 @@ namespace GitHub.Runner.Common
             _configFilePath = hostContext.GetConfigFile(WellKnownConfigFile.Runner);
             Trace.Info("ConfigFilePath: {0}", _configFilePath);
 
+            _migratedConfigFilePath = hostContext.GetConfigFile(WellKnownConfigFile.MigratedRunner);
+            Trace.Info("MigratedConfigFilePath: {0}", _migratedConfigFilePath);
+
             _credFilePath = hostContext.GetConfigFile(WellKnownConfigFile.Credentials);
             Trace.Info("CredFilePath: {0}", _credFilePath);
 
@@ -169,7 +177,7 @@ namespace GitHub.Runner.Common
         public bool HasCredentials()
         {
             Trace.Info("HasCredentials()");
-            bool credsStored = (new FileInfo(_credFilePath)).Exists || (new FileInfo(_migratedCredFilePath)).Exists;
+            bool credsStored = new FileInfo(_credFilePath).Exists || new FileInfo(_migratedCredFilePath).Exists;
             Trace.Info("stored {0}", credsStored);
             return credsStored;
         }
@@ -177,7 +185,7 @@ namespace GitHub.Runner.Common
         public bool IsConfigured()
         {
             Trace.Info("IsConfigured()");
-            bool configured = new FileInfo(_configFilePath).Exists;
+            bool configured = new FileInfo(_configFilePath).Exists || new FileInfo(_migratedConfigFilePath).Exists;
             Trace.Info("IsConfigured: {0}", configured);
             return configured;
         }
@@ -185,7 +193,7 @@ namespace GitHub.Runner.Common
         public bool IsServiceConfigured()
         {
             Trace.Info("IsServiceConfigured()");
-            bool serviceConfigured = (new FileInfo(_serviceConfigFilePath)).Exists;
+            bool serviceConfigured = new FileInfo(_serviceConfigFilePath).Exists;
             Trace.Info($"IsServiceConfigured: {serviceConfigured}");
             return serviceConfigured;
         }
@@ -229,6 +237,25 @@ namespace GitHub.Runner.Common
             return _settings;
         }
 
+        public RunnerSettings GetMigratedSettings()
+        {
+            if (_migratedSettings == null)
+            {
+                RunnerSettings configuredSettings = null;
+                if (File.Exists(_migratedConfigFilePath))
+                {
+                    string json = File.ReadAllText(_migratedConfigFilePath, Encoding.UTF8);
+                    Trace.Info($"Read migrated setting file: {json.Length} chars");
+                    configuredSettings = StringUtil.ConvertFromJson<RunnerSettings>(json);
+                }
+
+                ArgUtil.NotNull(configuredSettings, nameof(configuredSettings));
+                _migratedSettings = configuredSettings;
+            }
+
+            return _migratedSettings;
+        }
+
         public void SaveCredential(CredentialData credential)
         {
             Trace.Info("Saving {0} credential @ {1}", credential.Scheme, _credFilePath);
@@ -244,6 +271,21 @@ namespace GitHub.Runner.Common
             File.SetAttributes(_credFilePath, File.GetAttributes(_credFilePath) | FileAttributes.Hidden);
         }
 
+        public void SaveMigratedCredential(CredentialData credential)
+        {
+            Trace.Info("Saving {0} migrated credential @ {1}", credential.Scheme, _migratedCredFilePath);
+            if (File.Exists(_migratedCredFilePath))
+            {
+                // Delete existing credential file first, since the file is hidden and not able to overwrite.
+                Trace.Info("Delete exist runner migrated credential file.");
+                IOUtil.DeleteFile(_migratedCredFilePath);
+            }
+
+            IOUtil.SaveObject(credential, _migratedCredFilePath);
+            Trace.Info("Migrated Credentials Saved.");
+            File.SetAttributes(_migratedCredFilePath, File.GetAttributes(_migratedCredFilePath) | FileAttributes.Hidden);
+        }
+
         public void SaveSettings(RunnerSettings settings)
         {
             Trace.Info("Saving runner settings.");
@@ -259,6 +301,21 @@ namespace GitHub.Runner.Common
             File.SetAttributes(_configFilePath, File.GetAttributes(_configFilePath) | FileAttributes.Hidden);
         }
 
+        public void SaveMigratedSettings(RunnerSettings settings)
+        {
+            Trace.Info("Saving runner migrated settings");
+            if (File.Exists(_migratedConfigFilePath))
+            {
+                // Delete existing settings file first, since the file is hidden and not able to overwrite.
+                Trace.Info("Delete exist runner migrated settings file.");
+                IOUtil.DeleteFile(_migratedConfigFilePath);
+            }
+
+            IOUtil.SaveObject(settings, _migratedConfigFilePath);
+            Trace.Info("Migrated Settings Saved.");
+            File.SetAttributes(_migratedConfigFilePath, File.GetAttributes(_migratedConfigFilePath) | FileAttributes.Hidden);
+        }
+
         public void DeleteCredential()
         {
             IOUtil.Delete(_credFilePath, default(CancellationToken));
@@ -273,6 +330,12 @@ namespace GitHub.Runner.Common
         public void DeleteSettings()
         {
             IOUtil.Delete(_configFilePath, default(CancellationToken));
+            IOUtil.Delete(_migratedConfigFilePath, default(CancellationToken));
+        }
+
+        public void DeleteMigratedSettings()
+        {
+            IOUtil.Delete(_migratedConfigFilePath, default(CancellationToken));
         }
     }
 }

src/Runner.Common/Constants.cs

@@ -18,6 +18,7 @@ namespace GitHub.Runner.Common
     public enum WellKnownConfigFile
     {
         Runner,
+        MigratedRunner,
         Credentials,
         MigratedCredentials,
         RSACredentials,
@@ -153,13 +154,14 @@ namespace GitHub.Runner.Common
                 public const int RetryableError = 2;
                 public const int RunnerUpdating = 3;
                 public const int RunOnceRunnerUpdating = 4;
+                public const int SessionConflict = 5;
             }
 
             public static class Features
             {
                 public static readonly string DiskSpaceWarning = "runner.diskspace.warning";
-                public static readonly string Node16Warning = "DistributedTask.AddWarningToNode16Action";
                 public static readonly string LogTemplateErrorsAsDebugMessages = "DistributedTask.LogTemplateErrorsAsDebugMessages";
+                public static readonly string SkipRetryCompleteJobUponKnownErrors = "actions_skip_retry_complete_job_upon_known_errors";
                 public static readonly string UseContainerPathForTemplate = "DistributedTask.UseContainerPathForTemplate";
                 public static readonly string AllowRunnerContainerHooks = "DistributedTask.AllowRunnerContainerHooks";
             }
@@ -175,11 +177,6 @@ namespace GitHub.Runner.Common
             public static readonly string UnsupportedStopCommandTokenDisabled = "You cannot use a endToken that is an empty string, the string 'pause-logging', or another workflow command. For more information see: https://docs.github.com/actions/learn-github-actions/workflow-commands-for-github-actions#example-stopping-and-starting-workflow-commands or opt into insecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS` environment variable to `true`.";
             public static readonly string UnsupportedSummarySize = "$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of {0}k, got {1}k. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary";
             public static readonly string SummaryUploadError = "$GITHUB_STEP_SUMMARY upload aborted, an error occurred when uploading the summary. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary";
-            public static readonly string DetectedNodeAfterEndOfLifeMessage = "Node.js 16 actions are deprecated. Please update the following actions to use Node.js 20: {0}. For more information see: https://github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/.";
-            public static readonly string DeprecatedNodeDetectedAfterEndOfLifeActions = "DeprecatedNodeActionsMessageWarnings";
-            public static readonly string DeprecatedNodeVersion = "node16";
-            public static readonly string EnforcedNode12DetectedAfterEndOfLife = "The following actions uses node12 which is deprecated and will be forced to run on node16: {0}. For more info: https://github.blog/changelog/2023-06-13-github-actions-all-actions-will-run-on-node16-instead-of-node12-by-default/";
-            public static readonly string EnforcedNode12DetectedAfterEndOfLifeEnvVariable = "Node16ForceActionsWarnings";
         }
 
         public static class RunnerEvent
@@ -250,14 +247,13 @@ namespace GitHub.Runner.Common
                 public static readonly string RequireJobContainer = "ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER";
                 public static readonly string RunnerDebug = "ACTIONS_RUNNER_DEBUG";
                 public static readonly string StepDebug = "ACTIONS_STEP_DEBUG";
-                public static readonly string AllowActionsUseUnsecureNodeVersion = "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION";
             }
 
             public static class Agent
             {
                 public static readonly string ToolsDirectory = "agent.ToolsDirectory";
 
-                // Set this env var to "node12" to downgrade the node version for internal functions (e.g hashfiles). This does NOT affect the version of node actions.
+                // Set this env var to "nodeXY" to downgrade the node version for internal functions (e.g hashfiles). This does NOT affect the version of node actions.
                 public static readonly string ForcedInternalNodeVersion = "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION";
                 public static readonly string ForcedActionsNodeVersion = "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION";
                 public static readonly string PrintLogToStdout = "ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT";

src/Runner.Common/HostContext.cs

@@ -36,6 +36,7 @@ namespace GitHub.Runner.Common
         event EventHandler Unloading;
         void ShutdownRunner(ShutdownReason reason);
         void WritePerfCounter(string counter);
+        void LoadDefaultUserAgents();
     }
 
     public enum StartupType
@@ -67,6 +68,7 @@ namespace GitHub.Runner.Common
         private StartupType _startupType;
         private string _perfFile;
         private RunnerWebProxy _webProxy = new();
+        private string _hostType = string.Empty;
 
         public event EventHandler Unloading;
         public CancellationToken RunnerShutdownToken => _runnerShutdownTokenSource.Token;
@@ -78,6 +80,7 @@ namespace GitHub.Runner.Common
         {
             // Validate args.
             ArgUtil.NotNullOrEmpty(hostType, nameof(hostType));
+            _hostType = hostType;
 
             _loadContext = AssemblyLoadContext.GetLoadContext(typeof(HostContext).GetTypeInfo().Assembly);
             _loadContext.Unloading += LoadContext_Unloading;
@@ -196,6 +199,16 @@ namespace GitHub.Runner.Common
                 }
             }
 
+            if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY")))
+            {
+                _trace.Warning($"Runner is running under insecure mode: HTTPS server certificate validation has been turned off by GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY environment variable.");
+            }
+
+            LoadDefaultUserAgents();
+        }
+
+        public void LoadDefaultUserAgents()
+        {
             if (string.IsNullOrEmpty(WebProxy.HttpProxyAddress) && string.IsNullOrEmpty(WebProxy.HttpsProxyAddress))
             {
                 _trace.Info($"No proxy settings were found based on environmental variables (http_proxy/https_proxy/HTTP_PROXY/HTTPS_PROXY)");
@@ -205,11 +218,6 @@ namespace GitHub.Runner.Common
                 _userAgents.Add(new ProductInfoHeaderValue("HttpProxyConfigured", bool.TrueString));
             }
 
-            if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY")))
-            {
-                _trace.Warning($"Runner is running under insecure mode: HTTPS server certificate validation has been turned off by GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY environment variable.");
-            }
-
             var credFile = GetConfigFile(WellKnownConfigFile.Credentials);
             if (File.Exists(credFile))
             {
@@ -244,6 +252,11 @@ namespace GitHub.Runner.Common
                 _trace.Info($"Adding extra user agent '{extraUserAgentHeader}' to all HTTP requests.");
                 _userAgents.Add(extraUserAgentHeader);
             }
+
+            var currentProcess = Process.GetCurrentProcess();
+            _userAgents.Add(new ProductInfoHeaderValue("Pid", currentProcess.Id.ToString()));
+            _userAgents.Add(new ProductInfoHeaderValue("CreationTime", Uri.EscapeDataString(DateTime.UtcNow.ToString("O"))));
+            _userAgents.Add(new ProductInfoHeaderValue($"({_hostType})"));
         }
 
         public string GetDirectory(WellKnownDirectory directory)
@@ -330,6 +343,12 @@ namespace GitHub.Runner.Common
                         ".runner");
                     break;
 
+                case WellKnownConfigFile.MigratedRunner:
+                    path = Path.Combine(
+                        GetDirectory(WellKnownDirectory.Root),
+                        ".runner_migrated");
+                    break;
+
                 case WellKnownConfigFile.Credentials:
                     path = Path.Combine(
                         GetDirectory(WellKnownDirectory.Root),
@@ -616,7 +635,7 @@ namespace GitHub.Runner.Common
                     payload[0] = Enum.Parse(typeof(GitHub.Services.Common.VssCredentialsType), ((int)payload[0]).ToString());
                 }
 
-                if (payload.Length > 0)
+                if (payload.Length > 0 && !string.IsNullOrEmpty(eventData.Message))
                 {
                     message = String.Format(eventData.Message.Replace("%n", Environment.NewLine), payload);
                 }

src/Runner.Common/JobServer.cs

@@ -4,6 +4,7 @@ using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Headers;
+using System.Net.Security;
 using System.Net.WebSockets;
 using System.Text;
 using System.Threading;
@@ -179,6 +180,10 @@ namespace GitHub.Runner.Common
                     userAgentValues.AddRange(UserAgentUtility.GetDefaultRestUserAgent());
                     userAgentValues.AddRange(HostContext.UserAgents);
                     this._websocketClient.Options.SetRequestHeader("User-Agent", string.Join(" ", userAgentValues.Select(x => x.ToString())));
+                    if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY")))
+                    {
+                        this._websocketClient.Options.RemoteCertificateValidationCallback = (_, _, _, _) => true;
+                    }
 
                     this._websocketConnectTask = ConnectWebSocketClient(feedStreamUrl, delay);
                 }

src/Runner.Common/JobServerQueue.cs

@@ -19,7 +19,7 @@ namespace GitHub.Runner.Common
         TaskCompletionSource<int> JobRecordUpdated { get; }
         event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling;
         Task ShutdownAsync();
-        void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false, bool enableTelemetry = false);
+        void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false);
         void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null);
         void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource);
         void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines);
@@ -74,6 +74,7 @@ namespace GitHub.Runner.Common
         private readonly List<JobTelemetry> _jobTelemetries = new();
         private bool _queueInProcess = false;
         private bool _resultsServiceOnly = false;
+        private int _resultsServiceExceptionsCount = 0;
         private Stopwatch _resultsUploadTimer = new();
         private Stopwatch _actionsUploadTimer = new();
 
@@ -104,11 +105,10 @@ namespace GitHub.Runner.Common
             _resultsServer = hostContext.GetService<IResultsServer>();
         }
 
-        public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false, bool enableTelemetry = false)
+        public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false)
         {
             Trace.Entering();
             _resultsServiceOnly = resultsServiceOnly;
-            _enableTelemetry = enableTelemetry;
 
             var serviceEndPoint = jobRequest.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
 
@@ -139,6 +139,12 @@ namespace GitHub.Runner.Common
                 _resultsClientInitiated = true;
             }
 
+            // Enable telemetry if we have both results service and actions service
+            if (_resultsClientInitiated && !_resultsServiceOnly)
+            {
+                _enableTelemetry = true;
+            }
+
             if (_queueInProcess)
             {
                 Trace.Info("No-opt, all queue process tasks are running.");
@@ -551,6 +557,10 @@ namespace GitHub.Runner.Common
                             {
                                 await UploadSummaryFile(file);
                             }
+                            if (string.Equals(file.Type, CoreAttachmentType.ResultsDiagnosticLog, StringComparison.OrdinalIgnoreCase))
+                            {
+                                await UploadResultsDiagnosticLogsFile(file);
+                            }
                             else if (String.Equals(file.Type, CoreAttachmentType.ResultsLog, StringComparison.OrdinalIgnoreCase))
                             {
                                 if (file.RecordId != _jobTimelineRecordId)
@@ -570,9 +580,9 @@ namespace GitHub.Runner.Common
                             Trace.Info("Catch exception during file upload to results, keep going since the process is best effort.");
                             Trace.Error(ex);
                             errorCount++;
-
+                            _resultsServiceExceptionsCount++;
                             // If we hit any exceptions uploading to Results, let's skip any additional uploads to Results unless Results is serving logs
-                            if (!_resultsServiceOnly)
+                            if (!_resultsServiceOnly && _resultsServiceExceptionsCount > 3)
                             {
                                 _resultsClientInitiated = false;
                                 SendResultsTelemetry(ex);
@@ -603,7 +613,7 @@ namespace GitHub.Runner.Common
 
         private void SendResultsTelemetry(Exception ex)
         {
-            var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception with results. {ex.Message}" };
+            var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception with results. {HostContext.SecretMasker.MaskSecrets(ex.Message)}" };
             issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.ResultsUploadFailure;
 
             var telemetryRecord = new TimelineRecord()
@@ -699,7 +709,9 @@ namespace GitHub.Runner.Common
                             {
                                 Trace.Info("Catch exception during update steps, skip update Results.");
                                 Trace.Error(e);
-                                if (!_resultsServiceOnly)
+                                _resultsServiceExceptionsCount++;
+                                // If we hit any exceptions uploading to Results, let's skip any additional uploads to Results unless Results is serving logs
+                                if (!_resultsServiceOnly && _resultsServiceExceptionsCount > 3)
                                 {
                                     _resultsClientInitiated = false;
                                     SendResultsTelemetry(e);
@@ -922,6 +934,17 @@ namespace GitHub.Runner.Common
             await UploadResultsFile(file, summaryHandler);
         }
 
+        private async Task UploadResultsDiagnosticLogsFile(ResultsUploadFileInfo file)
+        {
+            Trace.Info($"Starting to upload diagnostic logs file to results service {file.Name}, {file.Path}");
+            ResultsFileUploadHandler diagnosticLogsHandler = async (file) =>
+            {
+                await _resultsServer.CreateResultsDiagnosticLogsAsync(file.PlanId, file.JobId, file.Path, CancellationToken.None);
+            };
+
+            await UploadResultsFile(file, diagnosticLogsHandler);
+        }
+
         private async Task UploadResultsStepLogFile(ResultsUploadFileInfo file)
         {
             Trace.Info($"Starting upload of step log file to results service {file.Name}, {file.Path}");

src/Runner.Common/LaunchServer.cs

@@ -1,11 +1,12 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
+using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Sdk;
+using GitHub.Services.Common;
 using GitHub.Services.Launch.Client;
-using GitHub.Services.WebApi;
 
 namespace GitHub.Runner.Common
 {
@@ -23,8 +24,21 @@ namespace GitHub.Runner.Common
 
         public void InitializeLaunchClient(Uri uri, string token)
         {
-            var httpMessageHandler = HostContext.CreateHttpClientHandler();
-            this._launchClient = new LaunchHttpClient(uri, httpMessageHandler, token, disposeHandler: true);
+            // Using default 100 timeout
+            RawClientHttpRequestSettings settings = VssUtil.GetHttpRequestSettings(null);
+
+            // Create retry handler
+            IEnumerable<DelegatingHandler> delegatingHandlers = new List<DelegatingHandler>();
+            if (settings.MaxRetryRequest > 0)
+            {
+                delegatingHandlers = new DelegatingHandler[] { new VssHttpRetryMessageHandler(settings.MaxRetryRequest) };
+            }
+
+            // Setup RawHttpMessageHandler without credentials
+            var httpMessageHandler = new RawHttpMessageHandler(new NoOpCredentials(null), settings);
+            var pipeline = HttpClientFactory.CreatePipeline(httpMessageHandler, delegatingHandlers);
+
+            this._launchClient = new LaunchHttpClient(uri, pipeline, token, disposeHandler: true);
         }
 
         public Task<ActionDownloadInfoCollection> ResolveActionsDownloadInfoAsync(Guid planId, Guid jobId, ActionReferenceList actionReferenceList,

src/Runner.Common/ResultsServer.cs

@@ -35,6 +35,8 @@ namespace GitHub.Runner.Common
 
         Task UpdateResultsWorkflowStepsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId,
             IEnumerable<TimelineRecord> records, CancellationToken cancellationToken);
+
+        Task CreateResultsDiagnosticLogsAsync(string planId, string jobId, string file, CancellationToken cancellationToken);
     }
 
     public sealed class ResultServer : RunnerService, IResultsServer
@@ -141,6 +143,18 @@ namespace GitHub.Runner.Common
             throw new InvalidOperationException("Results client is not initialized.");
         }
 
+        public Task CreateResultsDiagnosticLogsAsync(string planId, string jobId, string file,
+            CancellationToken cancellationToken)
+        {
+            if (_resultsClient != null)
+            {
+                return _resultsClient.UploadResultsDiagnosticLogsAsync(planId, jobId, file,
+                    cancellationToken: cancellationToken);
+            }
+
+            throw new InvalidOperationException("Results client is not initialized.");
+        }
+
         public ValueTask DisposeAsync()
         {
             CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None);

src/Runner.Common/RunServer.cs

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
 using GitHub.Actions.RunService.WebApi;
 using GitHub.DistributedTask.Pipelines;
 using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
 using Sdk.RSWebApi.Contracts;
@@ -17,7 +18,7 @@ namespace GitHub.Runner.Common
     {
         Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
 
-        Task<AgentJobRequestMessage> GetJobMessageAsync(string id, CancellationToken token);
+        Task<AgentJobRequestMessage> GetJobMessageAsync(string id, string billingOwnerId, CancellationToken token);
 
         Task CompleteJobAsync(
             Guid planId,
@@ -27,6 +28,20 @@ namespace GitHub.Runner.Common
             IList<StepResult> stepResults,
             IList<Annotation> jobAnnotations,
             string environmentUrl,
+            IList<Telemetry> telemetry,
+            string billingOwnerId,
+            CancellationToken token);
+
+        Task CompleteJob2Async(
+            Guid planId,
+            Guid jobId,
+            TaskResult result,
+            Dictionary<String, VariableValue> outputs,
+            IList<StepResult> stepResults,
+            IList<Annotation> jobAnnotations,
+            string environmentUrl,
+            IList<Telemetry> telemetry,
+            string billingOwnerId,
             CancellationToken token);
 
         Task<RenewJobResponse> RenewJobAsync(Guid planId, Guid jobId, CancellationToken token);
@@ -56,14 +71,18 @@ namespace GitHub.Runner.Common
             }
         }
 
-        public Task<AgentJobRequestMessage> GetJobMessageAsync(string id, CancellationToken cancellationToken)
+        public Task<AgentJobRequestMessage> GetJobMessageAsync(string id, string billingOwnerId, CancellationToken cancellationToken)
         {
             CheckConnection();
             return RetryRequest<AgentJobRequestMessage>(
-                async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, cancellationToken), cancellationToken,
-                shouldRetry: ex => ex is not TaskOrchestrationJobAlreadyAcquiredException);
+                async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, VarUtil.OS, billingOwnerId, cancellationToken), cancellationToken,
+                shouldRetry: ex =>
+                    ex is not TaskOrchestrationJobNotFoundException &&          // HTTP status 404
+                    ex is not TaskOrchestrationJobAlreadyAcquiredException &&   // HTTP status 409
+                    ex is not TaskOrchestrationJobUnprocessableException);      // HTTP status 422
         }
 
+        // Legacy will be deleted when SkipRetryCompleteJobUponKnownErrors is cleaned up
         public Task CompleteJobAsync(
             Guid planId,
             Guid jobId,
@@ -72,11 +91,33 @@ namespace GitHub.Runner.Common
             IList<StepResult> stepResults,
             IList<Annotation> jobAnnotations,
             string environmentUrl,
+            IList<Telemetry> telemetry,
+            string billingOwnerId,
             CancellationToken cancellationToken)
         {
             CheckConnection();
             return RetryRequest(
-                async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, cancellationToken), cancellationToken);
+                async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, telemetry, billingOwnerId, cancellationToken), cancellationToken);
+        }
+
+        public Task CompleteJob2Async(
+            Guid planId,
+            Guid jobId,
+            TaskResult result,
+            Dictionary<String, VariableValue> outputs,
+            IList<StepResult> stepResults,
+            IList<Annotation> jobAnnotations,
+            string environmentUrl,
+            IList<Telemetry> telemetry,
+            string billingOwnerId,
+            CancellationToken cancellationToken)
+        {
+            CheckConnection();
+            return RetryRequest(
+                async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, telemetry, billingOwnerId, cancellationToken), cancellationToken,
+                shouldRetry: ex =>
+                    ex is not VssUnauthorizedException &&               // HTTP status 401
+                    ex is not TaskOrchestrationJobNotFoundException);   // HTTP status 404
         }
 
         public Task<RenewJobResponse> RenewJobAsync(Guid planId, Guid jobId, CancellationToken cancellationToken)

src/Runner.Common/Runner.Common.csproj

@@ -1,11 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <OutputType>Library</OutputType>
     <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
     <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
     <Version>$(Version)</Version>
   </PropertyGroup>
 
@@ -15,11 +15,11 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
+    <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
-    <PackageReference Include="System.Text.Encoding.CodePages" Version="4.4.0" />
-    <PackageReference Include="System.Threading.Channels" Version="4.4.0" />
+    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
+    <PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
+    <PackageReference Include="System.Threading.Channels" Version="8.0.0" />
   </ItemGroup>
 
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">

src/Runner.Common/RunnerDotcomServer.cs

@@ -46,7 +46,11 @@ namespace GitHub.Runner.Common
             var githubApiUrl = "";
             var gitHubUrlBuilder = new UriBuilder(githubUrl);
             var path = gitHubUrlBuilder.Path.Split('/', '\\', StringSplitOptions.RemoveEmptyEntries);
-            if (path.Length == 1)
+            var isOrgRunner = path.Length == 1;
+            var isRepoOrEnterpriseRunner = path.Length == 2;
+            var isRepoRunner = isRepoOrEnterpriseRunner && !string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase);
+
+            if (isOrgRunner)
             {
                 // org runner
                 if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
@@ -58,21 +62,31 @@ namespace GitHub.Runner.Common
                     githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/orgs/{path[0]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
                 }
             }
-            else if (path.Length == 2)
+            else if (isRepoOrEnterpriseRunner)
             {
-                // repo or enterprise runner.
-                if (!string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase))
+                // Repository runner
+                if (isRepoRunner)
                 {
-                    return null;
-                }
-
-                if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
-                {
-                    githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
+                    if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/repos/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
+                    }
+                    else
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/repos/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
+                    }
                 }
                 else
                 {
-                    githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
+                    // Enterprise runner
+                    if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
+                    }
+                    else
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}";
+                    }
                 }
             }
             else
@@ -90,7 +104,11 @@ namespace GitHub.Runner.Common
             var githubApiUrl = "";
             var gitHubUrlBuilder = new UriBuilder(githubUrl);
             var path = gitHubUrlBuilder.Path.Split('/', '\\', StringSplitOptions.RemoveEmptyEntries);
-            if (path.Length == 1)
+            var isOrgRunner = path.Length == 1;
+            var isRepoOrEnterpriseRunner = path.Length == 2;
+            var isRepoRunner = isRepoOrEnterpriseRunner && !string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase);
+
+            if (isOrgRunner)
             {
                 // org runner
                 if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
@@ -102,21 +120,31 @@ namespace GitHub.Runner.Common
                     githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/orgs/{path[0]}/actions/runner-groups";
                 }
             }
-            else if (path.Length == 2)
+            else if (isRepoOrEnterpriseRunner)
             {
-                // repo or enterprise runner.
-                if (!string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase))
+                // Repository Runner
+                if (isRepoRunner)
                 {
-                    return null;
-                }
-
-                if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
-                {
-                    githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runner-groups";
+                    if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/repos/{path[0]}/{path[1]}/actions/runner-groups";
+                    }
+                    else
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/repos/{path[0]}/{path[1]}/actions/runner-groups";
+                    }
                 }
                 else
                 {
-                    githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runner-groups";
+                    // Enterprise Runner
+                    if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runner-groups";
+                    }
+                    else
+                    {
+                        githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runner-groups";
+                    }
                 }
             }
             else

src/Runner.Common/RunnerServer.cs

@@ -1,11 +1,11 @@
-using GitHub.DistributedTask.WebApi;
-using System;
+using System;
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
-using GitHub.Services.WebApi;
-using GitHub.Services.Common;
+using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Sdk;
+using GitHub.Services.Common;
+using GitHub.Services.WebApi;
 
 namespace GitHub.Runner.Common
 {
@@ -50,7 +50,10 @@ namespace GitHub.Runner.Common
         Task<PackageMetadata> GetPackageAsync(string packageType, string platform, string version, bool includeToken, CancellationToken cancellationToken);
 
         // agent update
-        Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, ulong agentId, string currentState, string trace);
+        Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, ulong agentId, string currentState, string trace, CancellationToken cancellationToken = default);
+
+        // runner config refresh
+        Task<string> RefreshRunnerConfigAsync(int agentId, string configType, string encodedRunnerConfig, CancellationToken cancellationToken);
     }
 
     public sealed class RunnerServer : RunnerService, IRunnerServer
@@ -315,10 +318,17 @@ namespace GitHub.Runner.Common
             return _genericTaskAgentClient.GetPackageAsync(packageType, platform, version, includeToken, cancellationToken: cancellationToken);
         }
 
-        public Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, ulong agentId, string currentState, string trace)
+        public Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, ulong agentId, string currentState, string trace, CancellationToken cancellationToken = default)
         {
             CheckConnection(RunnerConnectionType.Generic);
-            return _genericTaskAgentClient.UpdateAgentUpdateStateAsync(agentPoolId, agentId, currentState, trace);
+            return _genericTaskAgentClient.UpdateAgentUpdateStateAsync(agentPoolId, agentId, currentState, trace, cancellationToken: cancellationToken);
+        }
+
+        // runner config refresh
+        public Task<string> RefreshRunnerConfigAsync(int agentId, string configType, string encodedRunnerConfig, CancellationToken cancellationToken)
+        {
+            CheckConnection(RunnerConnectionType.Generic);
+            return _genericTaskAgentClient.RefreshRunnerConfigAsync(agentId, configType, encodedRunnerConfig, cancellationToken: cancellationToken);
         }
     }
 }

src/Runner.Common/RunnerService.cs

@@ -70,7 +70,8 @@ namespace GitHub.Runner.Common
 
         protected async Task RetryRequest(Func<Task> func,
             CancellationToken cancellationToken,
-            int maxRetryAttemptsCount = 5
+            int maxRetryAttemptsCount = 5,
+            Func<Exception, bool> shouldRetry = null
         )
         {
             async Task<Unit> wrappedFunc()
@@ -78,7 +79,7 @@ namespace GitHub.Runner.Common
                 await func();
                 return Unit.Value;
             }
-            await RetryRequest<Unit>(wrappedFunc, cancellationToken, maxRetryAttemptsCount);
+            await RetryRequest<Unit>(wrappedFunc, cancellationToken, maxRetryAttemptsCount, shouldRetry);
         }
 
         protected async Task<T> RetryRequest<T>(Func<Task<T>> func,

src/Runner.Common/Util/NodeUtil.cs

@@ -5,8 +5,8 @@ namespace GitHub.Runner.Common.Util
 {
     public static class NodeUtil
     {
-        private const string _defaultNodeVersion = "node16";
-        public static readonly ReadOnlyCollection<string> BuiltInNodeVersions = new(new[] { "node16", "node20" });
+        private const string _defaultNodeVersion = "node20";
+        public static readonly ReadOnlyCollection<string> BuiltInNodeVersions = new(new[] { "node20" });
         public static string GetInternalNodeVersion()
         {
             var forcedInternalNodeVersion = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ForcedInternalNodeVersion);

src/Runner.Listener/BrokerMessageListener.cs

@@ -9,11 +9,12 @@ using System.Threading;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
+using GitHub.Runner.Common.Util;
 using GitHub.Runner.Listener.Configuration;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
-using GitHub.Runner.Common.Util;
 using GitHub.Services.OAuth;
+using GitHub.Services.WebApi;
 
 namespace GitHub.Runner.Listener
 {
@@ -24,25 +25,170 @@ namespace GitHub.Runner.Listener
         private TimeSpan _getNextMessageRetryInterval;
         private TaskAgentStatus runnerStatus = TaskAgentStatus.Online;
         private CancellationTokenSource _getMessagesTokenSource;
+        private VssCredentials _creds;
+        private TaskAgentSession _session;
+        private IRunnerServer _runnerServer;
         private IBrokerServer _brokerServer;
+        private readonly Dictionary<string, int> _sessionCreationExceptionTracker = new();
+        private bool _accessTokenRevoked = false;
+        private readonly TimeSpan _sessionCreationRetryInterval = TimeSpan.FromSeconds(30);
+        private readonly TimeSpan _sessionConflictRetryLimit = TimeSpan.FromMinutes(4);
+        private readonly TimeSpan _clockSkewRetryLimit = TimeSpan.FromMinutes(30);
+
 
         public override void Initialize(IHostContext hostContext)
         {
             base.Initialize(hostContext);
 
             _term = HostContext.GetService<ITerminal>();
+            _runnerServer = HostContext.GetService<IRunnerServer>();
             _brokerServer = HostContext.GetService<IBrokerServer>();
         }
 
-        public async Task<Boolean> CreateSessionAsync(CancellationToken token)
+        public async Task<CreateSessionResult> CreateSessionAsync(CancellationToken token)
         {
-            await RefreshBrokerConnection();
-            return await Task.FromResult(true);
+            Trace.Entering();
+
+            // Settings
+            var configManager = HostContext.GetService<IConfigurationManager>();
+            _settings = configManager.LoadSettings();
+            var serverUrlV2 = _settings.ServerUrlV2;
+            var serverUrl = _settings.ServerUrl;
+            Trace.Info(_settings);
+
+            if (string.IsNullOrEmpty(_settings.ServerUrlV2))
+            {
+                throw new InvalidOperationException("ServerUrlV2 is not set");
+            }
+
+            // Create connection.
+            Trace.Info("Loading Credentials");
+            var credMgr = HostContext.GetService<ICredentialManager>();
+            _creds = credMgr.LoadCredentials();
+
+            var agent = new TaskAgentReference
+            {
+                Id = _settings.AgentId,
+                Name = _settings.AgentName,
+                Version = BuildConstants.RunnerPackage.Version,
+                OSDescription = RuntimeInformation.OSDescription,
+            };
+            var currentProcess = Process.GetCurrentProcess();
+            string sessionName = $"{Environment.MachineName ?? "RUNNER"} (PID: {currentProcess.Id})";
+            var taskAgentSession = new TaskAgentSession(sessionName, agent);
+
+            string errorMessage = string.Empty;
+            bool encounteringError = false;
+
+            while (true)
+            {
+                token.ThrowIfCancellationRequested();
+                Trace.Info($"Attempt to create session.");
+                try
+                {
+                    Trace.Info("Connecting to the Broker Server...");
+                    await _brokerServer.ConnectAsync(new Uri(serverUrlV2), _creds);
+                    Trace.Info("VssConnection created");
+
+                    if (!string.IsNullOrEmpty(serverUrl) &&
+                        !string.Equals(serverUrl, serverUrlV2, StringComparison.OrdinalIgnoreCase))
+                    {
+                        Trace.Info("Connecting to the Runner server...");
+                        await _runnerServer.ConnectAsync(new Uri(serverUrl), _creds);
+                        Trace.Info("VssConnection created");
+                    }
+
+                    _term.WriteLine();
+                    _term.WriteSuccessMessage("Connected to GitHub");
+                    _term.WriteLine();
+
+                    _session = await _brokerServer.CreateSessionAsync(taskAgentSession, token);
+
+                    Trace.Info($"Session created.");
+                    if (encounteringError)
+                    {
+                        _term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
+                        _sessionCreationExceptionTracker.Clear();
+                        encounteringError = false;
+                    }
+
+                    return CreateSessionResult.Success;
+                }
+                catch (OperationCanceledException) when (token.IsCancellationRequested)
+                {
+                    Trace.Info("Session creation has been cancelled.");
+                    throw;
+                }
+                catch (TaskAgentAccessTokenExpiredException)
+                {
+                    Trace.Info("Runner OAuth token has been revoked. Session creation failed.");
+                    _accessTokenRevoked = true;
+                    throw;
+                }
+                catch (Exception ex)
+                {
+                    Trace.Error("Catch exception during create session.");
+                    Trace.Error(ex);
+
+                    if (ex is VssOAuthTokenRequestException vssOAuthEx && _creds.Federated is VssOAuthCredential vssOAuthCred)
+                    {
+                        // "invalid_client" means the runner registration has been deleted from the server.
+                        if (string.Equals(vssOAuthEx.Error, "invalid_client", StringComparison.OrdinalIgnoreCase))
+                        {
+                            _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
+                            return CreateSessionResult.Failure;
+                        }
+
+                        // Check whether we get 401 because the runner registration already removed by the service.
+                        // If the runner registration get deleted, we can't exchange oauth token.
+                        Trace.Error("Test oauth app registration.");
+                        var oauthTokenProvider = new VssOAuthTokenProvider(vssOAuthCred, new Uri(serverUrlV2));
+                        var authError = await oauthTokenProvider.ValidateCredentialAsync(token);
+                        if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase))
+                        {
+                            _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
+                            return CreateSessionResult.Failure;
+                        }
+                    }
+
+                    if (!IsSessionCreationExceptionRetriable(ex))
+                    {
+                        _term.WriteError($"Failed to create session. {ex.Message}");
+                        if (ex is TaskAgentSessionConflictException)
+                        {
+                            return CreateSessionResult.SessionConflict;
+                        }
+                        return CreateSessionResult.Failure;
+                    }
+
+                    if (!encounteringError) //print the message only on the first error
+                    {
+                        _term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
+                        encounteringError = true;
+                    }
+
+                    Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
+                    await HostContext.Delay(_sessionCreationRetryInterval, token);
+                }
+            }
         }
 
         public async Task DeleteSessionAsync()
         {
-            await Task.CompletedTask;
+            if (_session != null && _session.SessionId != Guid.Empty)
+            {
+                if (!_accessTokenRevoked)
+                {
+                    using (var ts = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
+                    {
+                        await _brokerServer.DeleteSessionAsync(ts.Token);
+                    }
+                }
+                else
+                {
+                    Trace.Warning("Runner OAuth token has been revoked. Skip deleting session.");
+                }
+            }
         }
 
         public void OnJobStatus(object sender, JobStatusEventArgs e)
@@ -73,12 +219,13 @@ namespace GitHub.Runner.Listener
                 _getMessagesTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
                 try
                 {
-                    message = await _brokerServer.GetRunnerMessageAsync(_getMessagesTokenSource.Token,
+                    message = await _brokerServer.GetRunnerMessageAsync(_session.SessionId,
                                                                         runnerStatus,
                                                                         BuildConstants.RunnerPackage.Version,
                                                                         VarUtil.OS,
                                                                         VarUtil.OSArchitecture,
-                                                                        _settings.DisableUpdate);
+                                                                        _settings.DisableUpdate,
+                                                                        _getMessagesTokenSource.Token);
 
                     if (message == null)
                     {
@@ -102,10 +249,19 @@ namespace GitHub.Runner.Listener
                     Trace.Info("Runner OAuth token has been revoked. Unable to pull message.");
                     throw;
                 }
+                catch (HostedRunnerDeprovisionedException)
+                {
+                    Trace.Info("Hosted runner has been deprovisioned.");
+                    throw;
+                }
                 catch (AccessDeniedException e) when (e.ErrorCode == 1)
                 {
                     throw;
                 }
+                catch (RunnerNotFoundException)
+                {
+                    throw;
+                }
                 catch (Exception ex)
                 {
                     Trace.Error("Catch exception during get next message.");
@@ -143,7 +299,7 @@ namespace GitHub.Runner.Listener
                         }
 
                         // re-create VssConnection before next retry
-                        await RefreshBrokerConnection();
+                        await RefreshBrokerConnectionAsync();
 
                         Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
                         await HostContext.Delay(_getNextMessageRetryInterval, token);
@@ -173,6 +329,11 @@ namespace GitHub.Runner.Listener
             }
         }
 
+        public async Task RefreshListenerTokenAsync(CancellationToken cancellationToken)
+        {
+            await RefreshBrokerConnectionAsync();
+        }
+
         public async Task DeleteMessageAsync(TaskAgentMessage message)
         {
             await Task.CompletedTask;
@@ -184,6 +345,7 @@ namespace GitHub.Runner.Listener
                 ex is TaskAgentPoolNotFoundException ||
                 ex is TaskAgentSessionExpiredException ||
                 ex is AccessDeniedException ||
+                ex is RunnerNotFoundException ||
                 ex is VssUnauthorizedException)
             {
                 Trace.Info($"Non-retriable exception: {ex.Message}");
@@ -196,12 +358,84 @@ namespace GitHub.Runner.Listener
             }
         }
 
-        private async Task RefreshBrokerConnection()
+        private bool IsSessionCreationExceptionRetriable(Exception ex)
+        {
+            if (ex is TaskAgentNotFoundException)
+            {
+                Trace.Info("The runner no longer exists on the server. Stopping the runner.");
+                _term.WriteError("The runner no longer exists on the server. Please reconfigure the runner.");
+                return false;
+            }
+            else if (ex is TaskAgentSessionConflictException)
+            {
+                Trace.Info("The session for this runner already exists.");
+                _term.WriteError("A session for this runner already exists.");
+                if (_sessionCreationExceptionTracker.ContainsKey(nameof(TaskAgentSessionConflictException)))
+                {
+                    _sessionCreationExceptionTracker[nameof(TaskAgentSessionConflictException)]++;
+                    if (_sessionCreationExceptionTracker[nameof(TaskAgentSessionConflictException)] * _sessionCreationRetryInterval.TotalSeconds >= _sessionConflictRetryLimit.TotalSeconds)
+                    {
+                        Trace.Info("The session conflict exception have reached retry limit.");
+                        _term.WriteError($"Stop retry on SessionConflictException after retried for {_sessionConflictRetryLimit.TotalSeconds} seconds.");
+                        return false;
+                    }
+                }
+                else
+                {
+                    _sessionCreationExceptionTracker[nameof(TaskAgentSessionConflictException)] = 1;
+                }
+
+                Trace.Info("The session conflict exception haven't reached retry limit.");
+                return true;
+            }
+            else if (ex is VssOAuthTokenRequestException && ex.Message.Contains("Current server time is"))
+            {
+                Trace.Info("Local clock might be skewed.");
+                _term.WriteError("The local machine's clock may be out of sync with the server time by more than five minutes. Please sync your clock with your domain or internet time and try again.");
+                if (_sessionCreationExceptionTracker.ContainsKey(nameof(VssOAuthTokenRequestException)))
+                {
+                    _sessionCreationExceptionTracker[nameof(VssOAuthTokenRequestException)]++;
+                    if (_sessionCreationExceptionTracker[nameof(VssOAuthTokenRequestException)] * _sessionCreationRetryInterval.TotalSeconds >= _clockSkewRetryLimit.TotalSeconds)
+                    {
+                        Trace.Info("The OAuth token request exception have reached retry limit.");
+                        _term.WriteError($"Stopped retrying OAuth token request exception after {_clockSkewRetryLimit.TotalSeconds} seconds.");
+                        return false;
+                    }
+                }
+                else
+                {
+                    _sessionCreationExceptionTracker[nameof(VssOAuthTokenRequestException)] = 1;
+                }
+
+                Trace.Info("The OAuth token request exception haven't reached retry limit.");
+                return true;
+            }
+            else if (ex is TaskAgentPoolNotFoundException ||
+                     ex is AccessDeniedException ||
+                     ex is VssUnauthorizedException)
+            {
+                Trace.Info($"Non-retriable exception: {ex.Message}");
+                return false;
+            }
+
+            else if (ex is InvalidOperationException)
+            {
+                Trace.Info($"Non-retriable exception: {ex.Message}");
+                return false;
+            }
+            else
+            {
+                Trace.Info($"Retriable exception: {ex.Message}");
+                return true;
+            }
+        }
+
+        private async Task RefreshBrokerConnectionAsync()
         {
             var configManager = HostContext.GetService<IConfigurationManager>();
             _settings = configManager.LoadSettings();
 
-            if (_settings.ServerUrlV2 == null)
+            if (string.IsNullOrEmpty(_settings.ServerUrlV2))
             {
                 throw new InvalidOperationException("ServerUrlV2 is not set");
             }

src/Runner.Listener/Configuration/ConfigurationManager.cs

@@ -404,6 +404,20 @@ namespace GitHub.Runner.Listener.Configuration
                 }
             }
 
+            // allow the server to override the serverUrlV2 and useV2Flow
+            if (agent.Properties.TryGetValue("ServerUrlV2", out string serverUrlV2) &&
+                !string.IsNullOrEmpty(serverUrlV2))
+            {
+                Trace.Info($"Service enforced serverUrlV2: {serverUrlV2}");
+                runnerSettings.ServerUrlV2 = serverUrlV2;
+            }
+
+            if (agent.Properties.TryGetValue("UseV2Flow", out bool useV2Flow) && useV2Flow)
+            {
+                Trace.Info($"Service enforced useV2Flow: {useV2Flow}");
+                runnerSettings.UseV2Flow = useV2Flow;
+            }
+
             _term.WriteSection("Runner settings");
 
             // We will Combine() what's stored with root.  Defaults to string a relative path

src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs

@@ -1,4 +1,5 @@
 #if OS_WINDOWS
+#pragma warning disable CA1416
 using System.IO;
 using System.Security.Cryptography;
 using System.Text;
@@ -84,4 +85,5 @@ namespace GitHub.Runner.Listener.Configuration
         }
     }
 }
+#pragma warning restore CA1416
 #endif

src/Runner.Listener/ErrorThrottler.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.Runner.Common;
+using GitHub.Services.Common;
+
+namespace GitHub.Runner.Listener
+{
+    [ServiceLocator(Default = typeof(ErrorThrottler))]
+    public interface IErrorThrottler : IRunnerService
+    {
+        void Reset();
+        Task IncrementAndWaitAsync(CancellationToken token);
+    }
+
+    public sealed class ErrorThrottler : RunnerService, IErrorThrottler
+    {
+        internal static readonly TimeSpan MinBackoff = TimeSpan.FromSeconds(1);
+        internal static readonly TimeSpan MaxBackoff = TimeSpan.FromMinutes(1);
+        internal static readonly TimeSpan BackoffCoefficient = TimeSpan.FromSeconds(1);
+        private int _count = 0;
+
+        public void Reset()
+        {
+            _count = 0;
+        }
+
+        public async Task IncrementAndWaitAsync(CancellationToken token)
+        {
+            if (++_count <= 1)
+            {
+                return;
+            }
+
+            TimeSpan backoff = BackoffTimerHelper.GetExponentialBackoff(
+                attempt: _count - 2, // 0-based attempt
+                minBackoff: MinBackoff,
+                maxBackoff: MaxBackoff,
+                deltaBackoff: BackoffCoefficient);
+            Trace.Warning($"Back off {backoff.TotalSeconds} seconds before next attempt. Current consecutive error count: {_count}");
+            await HostContext.Delay(backoff, token);
+        }
+    }
+}

src/Runner.Listener/JobDispatcher.cs

@@ -35,7 +35,7 @@ namespace GitHub.Runner.Listener
     // This implementation of IJobDispatcher is not thread safe.
     // It is based on the fact that the current design of the runner is a dequeue
     // and processes one message from the message queue at a time.
-    // In addition, it only executes one job every time, 
+    // In addition, it only executes one job every time,
     // and the server will not send another job while this one is still running.
     public sealed class JobDispatcher : RunnerService, IJobDispatcher
     {
@@ -545,14 +545,36 @@ namespace GitHub.Runner.Listener
                                     detailInfo = string.Join(Environment.NewLine, workerOutput);
                                     Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
 
-                                    var jobServer = await InitializeJobServerAsync(systemConnection);
-                                    await LogWorkerProcessUnhandledException(jobServer, message, detailInfo);
-
-                                    // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
-                                    if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase))
+                                    try
                                     {
-                                        Trace.Info($"Finish job with result 'Failed' due to IOException.");
-                                        await ForceFailJob(jobServer, message, detailInfo);
+                                        var jobServer = await InitializeJobServerAsync(systemConnection);
+                                        var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
+                                        unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
+                                        switch (jobServer)
+                                        {
+                                            case IJobServer js:
+                                                {
+                                                    await LogWorkerProcessUnhandledException(js, message, unhandledExceptionIssue);
+                                                    // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
+                                                    if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase))
+                                                    {
+                                                        Trace.Info($"Finish job with result 'Failed' due to IOException.");
+                                                        await ForceFailJob(js, message);
+                                                    }
+
+                                                    break;
+                                                }
+                                            case IRunServer rs:
+                                                await ForceFailJob(rs, message, unhandledExceptionIssue);
+                                                break;
+                                            default:
+                                                throw new NotSupportedException($"JobServer type '{jobServer.GetType().Name}' is not supported.");
+                                        }
+                                    }
+                                    catch (Exception ex)
+                                    {
+                                        Trace.Error($"Catch exception during log worker process unhandled exception.");
+                                        Trace.Error(ex);
                                     }
                                 }
 
@@ -629,8 +651,22 @@ namespace GitHub.Runner.Listener
                                     Trace.Info("worker process has been killed.");
                                 }
                             }
+                            catch (Exception ex)
+                            {
+                                // message send failed, this might indicate worker process is already exited or stuck.
+                                Trace.Info($"Job cancel message sending for job {message.JobId} failed, kill running worker. {ex}");
+                                workerProcessCancelTokenSource.Cancel();
+                                try
+                                {
+                                    await workerProcessTask;
+                                }
+                                catch (OperationCanceledException)
+                                {
+                                    Trace.Info("worker process has been killed.");
+                                }
+                            }
 
-                            // wait worker to exit 
+                            // wait worker to exit
                             // if worker doesn't exit within timeout, then kill worker.
                             completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));
 
@@ -1117,86 +1153,65 @@ namespace GitHub.Runner.Listener
         }
 
         // log an error issue to job level timeline record
-        private async Task LogWorkerProcessUnhandledException(IRunnerService server, Pipelines.AgentJobRequestMessage message, string detailInfo)
+        private async Task LogWorkerProcessUnhandledException(IJobServer jobServer, Pipelines.AgentJobRequestMessage message, Issue issue)
         {
-            if (server is IJobServer jobServer)
+            try
             {
-                try
-                {
-                    var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None);
-                    ArgUtil.NotNull(timeline, nameof(timeline));
+                var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None);
+                ArgUtil.NotNull(timeline, nameof(timeline));
 
-                    TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job");
-                    ArgUtil.NotNull(jobRecord, nameof(jobRecord));
+                TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job");
+                ArgUtil.NotNull(jobRecord, nameof(jobRecord));
 
-                    var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
-                    unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
-                    jobRecord.ErrorCount++;
-                    jobRecord.Issues.Add(unhandledExceptionIssue);
+                jobRecord.ErrorCount++;
+                jobRecord.Issues.Add(issue);
 
-                    if (message.Variables.TryGetValue("DistributedTask.MarkJobAsFailedOnWorkerCrash", out var markJobAsFailedOnWorkerCrash) &&
-                        StringUtil.ConvertToBoolean(markJobAsFailedOnWorkerCrash?.Value))
-                    {
-                        Trace.Info("Mark the job as failed since the worker crashed");
-                        jobRecord.Result = TaskResult.Failed;
-                        // mark the job as completed so service will pickup the result
-                        jobRecord.State = TimelineRecordState.Completed;
-                    }
+                Trace.Info("Mark the job as failed since the worker crashed");
+                jobRecord.Result = TaskResult.Failed;
+                // mark the job as completed so service will pickup the result
+                jobRecord.State = TimelineRecordState.Completed;
 
-                    await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Error("Fail to report unhandled exception from Runner.Worker process");
-                    Trace.Error(ex);
-                }
+                await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None);
             }
-            else
+            catch (Exception ex)
             {
-                Trace.Info("Job server does not support handling unhandled exception yet, error message: {0}", detailInfo);
-                return;
+                Trace.Error("Fail to report unhandled exception from Runner.Worker process");
+                Trace.Error(ex);
             }
         }
 
         // raise job completed event to fail the job.
-        private async Task ForceFailJob(IRunnerService server, Pipelines.AgentJobRequestMessage message, string detailInfo)
+        private async Task ForceFailJob(IJobServer jobServer, Pipelines.AgentJobRequestMessage message)
         {
-            if (server is IJobServer jobServer)
+            try
             {
-                try
-                {
-                    var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, TaskResult.Failed);
-                    await jobServer.RaisePlanEventAsync<JobCompletedEvent>(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, CancellationToken.None);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Error("Fail to raise JobCompletedEvent back to service.");
-                    Trace.Error(ex);
-                }
+                var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, TaskResult.Failed);
+                await jobServer.RaisePlanEventAsync<JobCompletedEvent>(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, CancellationToken.None);
             }
-            else if (server is IRunServer runServer)
+            catch (Exception ex)
             {
-                try
-                {
-                    var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
-                    var unhandledAnnotation = unhandledExceptionIssue.ToAnnotation();
-                    var jobAnnotations = new List<Annotation>();
-                    if (unhandledAnnotation.HasValue)
-                    {
-                        jobAnnotations.Add(unhandledAnnotation.Value);
-                    }
+                Trace.Error("Fail to raise JobCompletedEvent back to service.");
+                Trace.Error(ex);
+            }
+        }
 
-                    await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, CancellationToken.None);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Error("Fail to raise job completion back to service.");
-                    Trace.Error(ex);
-                }
-            }
-            else
+        private async Task ForceFailJob(IRunServer runServer, Pipelines.AgentJobRequestMessage message, Issue issue)
+        {
+            try
             {
-                throw new NotSupportedException($"Server type {server.GetType().FullName} is not supported.");
+                var annotation = issue.ToAnnotation();
+                var jobAnnotations = new List<Annotation>();
+                if (annotation.HasValue)
+                {
+                    jobAnnotations.Add(annotation.Value);
+                }
+
+                await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, telemetry: null, billingOwnerId: message.BillingOwnerId, CancellationToken.None);
+            }
+            catch (Exception ex)
+            {
+                Trace.Error("Fail to raise job completion back to service.");
+                Trace.Error(ex);
             }
         }
 

src/Runner.Listener/MessageListener.cs

@@ -14,16 +14,26 @@ using GitHub.Runner.Listener.Configuration;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
 using GitHub.Services.OAuth;
+using GitHub.Services.WebApi;
 
 namespace GitHub.Runner.Listener
 {
+    public enum CreateSessionResult
+    {
+        Success,
+        Failure,
+        SessionConflict
+    }
+
     [ServiceLocator(Default = typeof(MessageListener))]
     public interface IMessageListener : IRunnerService
     {
-        Task<Boolean> CreateSessionAsync(CancellationToken token);
+        Task<CreateSessionResult> CreateSessionAsync(CancellationToken token);
         Task DeleteSessionAsync();
         Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token);
         Task DeleteMessageAsync(TaskAgentMessage message);
+
+        Task RefreshListenerTokenAsync(CancellationToken token);
         void OnJobStatus(object sender, JobStatusEventArgs e);
     }
 
@@ -33,6 +43,7 @@ namespace GitHub.Runner.Listener
         private RunnerSettings _settings;
         private ITerminal _term;
         private IRunnerServer _runnerServer;
+        private IBrokerServer _brokerServer;
         private TaskAgentSession _session;
         private TimeSpan _getNextMessageRetryInterval;
         private bool _accessTokenRevoked = false;
@@ -42,6 +53,9 @@ namespace GitHub.Runner.Listener
         private readonly Dictionary<string, int> _sessionCreationExceptionTracker = new();
         private TaskAgentStatus runnerStatus = TaskAgentStatus.Online;
         private CancellationTokenSource _getMessagesTokenSource;
+        private VssCredentials _creds;
+
+        private bool _isBrokerSession = false;
 
         public override void Initialize(IHostContext hostContext)
         {
@@ -49,9 +63,10 @@ namespace GitHub.Runner.Listener
 
             _term = HostContext.GetService<ITerminal>();
             _runnerServer = HostContext.GetService<IRunnerServer>();
+            _brokerServer = hostContext.GetService<IBrokerServer>();
         }
 
-        public async Task<Boolean> CreateSessionAsync(CancellationToken token)
+        public async Task<CreateSessionResult> CreateSessionAsync(CancellationToken token)
         {
             Trace.Entering();
 
@@ -64,7 +79,7 @@ namespace GitHub.Runner.Listener
             // Create connection.
             Trace.Info("Loading Credentials");
             var credMgr = HostContext.GetService<ICredentialManager>();
-            VssCredentials creds = credMgr.LoadCredentials();
+            _creds = credMgr.LoadCredentials();
 
             var agent = new TaskAgentReference
             {
@@ -73,7 +88,8 @@ namespace GitHub.Runner.Listener
                 Version = BuildConstants.RunnerPackage.Version,
                 OSDescription = RuntimeInformation.OSDescription,
             };
-            string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
+            var currentProcess = Process.GetCurrentProcess();
+            string sessionName = $"{Environment.MachineName ?? "RUNNER"} (PID: {currentProcess.Id})";
             var taskAgentSession = new TaskAgentSession(sessionName, agent);
 
             string errorMessage = string.Empty;
@@ -86,7 +102,7 @@ namespace GitHub.Runner.Listener
                 try
                 {
                     Trace.Info("Connecting to the Runner Server...");
-                    await _runnerServer.ConnectAsync(new Uri(serverUrl), creds);
+                    await _runnerServer.ConnectAsync(new Uri(serverUrl), _creds);
                     Trace.Info("VssConnection created");
 
                     _term.WriteLine();
@@ -98,6 +114,15 @@ namespace GitHub.Runner.Listener
                                                         taskAgentSession,
                                                         token);
 
+                    if (_session.BrokerMigrationMessage != null)
+                    {
+                        Trace.Info("Runner session is in migration mode: Creating Broker session with BrokerBaseUrl: {0}", _session.BrokerMigrationMessage.BrokerBaseUrl);
+
+                        await _brokerServer.UpdateConnectionIfNeeded(_session.BrokerMigrationMessage.BrokerBaseUrl, _creds);
+                        _session = await _brokerServer.CreateSessionAsync(taskAgentSession, token);
+                        _isBrokerSession = true;
+                    }
+
                     Trace.Info($"Session created.");
                     if (encounteringError)
                     {
@@ -106,7 +131,7 @@ namespace GitHub.Runner.Listener
                         encounteringError = false;
                     }
 
-                    return true;
+                    return CreateSessionResult.Success;
                 }
                 catch (OperationCanceledException) when (token.IsCancellationRequested)
                 {
@@ -124,13 +149,13 @@ namespace GitHub.Runner.Listener
                     Trace.Error("Catch exception during create session.");
                     Trace.Error(ex);
 
-                    if (ex is VssOAuthTokenRequestException vssOAuthEx && creds.Federated is VssOAuthCredential vssOAuthCred)
+                    if (ex is VssOAuthTokenRequestException vssOAuthEx && _creds.Federated is VssOAuthCredential vssOAuthCred)
                     {
                         // "invalid_client" means the runner registration has been deleted from the server.
                         if (string.Equals(vssOAuthEx.Error, "invalid_client", StringComparison.OrdinalIgnoreCase))
                         {
                             _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
-                            return false;
+                            return CreateSessionResult.Failure;
                         }
 
                         // Check whether we get 401 because the runner registration already removed by the service.
@@ -141,14 +166,18 @@ namespace GitHub.Runner.Listener
                         if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase))
                         {
                             _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
-                            return false;
+                            return CreateSessionResult.Failure;
                         }
                     }
 
                     if (!IsSessionCreationExceptionRetriable(ex))
                     {
                         _term.WriteError($"Failed to create session. {ex.Message}");
-                        return false;
+                        if (ex is TaskAgentSessionConflictException)
+                        {
+                            return CreateSessionResult.SessionConflict;
+                        }
+                        return CreateSessionResult.Failure;
                     }
 
                     if (!encounteringError) //print the message only on the first error
@@ -172,6 +201,11 @@ namespace GitHub.Runner.Listener
                     using (var ts = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
                     {
                         await _runnerServer.DeleteAgentSessionAsync(_settings.PoolId, _session.SessionId, ts.Token);
+
+                        if (_isBrokerSession)
+                        {
+                            await _brokerServer.DeleteSessionAsync(ts.Token);
+                        }
                     }
                 }
                 else
@@ -183,19 +217,17 @@ namespace GitHub.Runner.Listener
 
         public void OnJobStatus(object sender, JobStatusEventArgs e)
         {
-            if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("USE_BROKER_FLOW")))
+            Trace.Info("Received job status event. JobState: {0}", e.Status);
+            runnerStatus = e.Status;
+            try
             {
-                Trace.Info("Received job status event. JobState: {0}", e.Status);
-                runnerStatus = e.Status;
-                try
-                {
-                    _getMessagesTokenSource?.Cancel();
-                }
-                catch (ObjectDisposedException)
-                {
-                    Trace.Info("_getMessagesTokenSource is already disposed.");
-                }
+                _getMessagesTokenSource?.Cancel();
             }
+            catch (ObjectDisposedException)
+            {
+                Trace.Info("_getMessagesTokenSource is already disposed.");
+            }
+
         }
 
         public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
@@ -205,6 +237,7 @@ namespace GitHub.Runner.Listener
             ArgUtil.NotNull(_settings, nameof(_settings));
             bool encounteringError = false;
             int continuousError = 0;
+            int continuousEmptyMessage = 0;
             string errorMessage = string.Empty;
             Stopwatch heartbeat = new();
             heartbeat.Restart();
@@ -228,6 +261,21 @@ namespace GitHub.Runner.Listener
                     // Decrypt the message body if the session is using encryption
                     message = DecryptMessage(message);
 
+
+                    if (message != null && message.MessageType == BrokerMigrationMessage.MessageType)
+                    {
+                        var migrationMessage = JsonUtility.FromString<BrokerMigrationMessage>(message.Body);
+
+                        await _brokerServer.UpdateConnectionIfNeeded(migrationMessage.BrokerBaseUrl, _creds);
+                        message = await _brokerServer.GetRunnerMessageAsync(_session.SessionId,
+                                                                        runnerStatus,
+                                                                        BuildConstants.RunnerPackage.Version,
+                                                                        VarUtil.OS,
+                                                                        VarUtil.OSArchitecture,
+                                                                        _settings.DisableUpdate,
+                                                                        token);
+                    }
+
                     if (message != null)
                     {
                         _lastMessageId = message.MessageId;
@@ -256,17 +304,26 @@ namespace GitHub.Runner.Listener
                     _accessTokenRevoked = true;
                     throw;
                 }
+                catch (HostedRunnerDeprovisionedException)
+                {
+                    Trace.Info("Hosted runner has been deprovisioned.");
+                    throw;
+                }
                 catch (AccessDeniedException e) when (e.ErrorCode == 1)
                 {
                     throw;
                 }
+                catch (RunnerNotFoundException)
+                {
+                    throw;
+                }
                 catch (Exception ex)
                 {
                     Trace.Error("Catch exception during get next message.");
                     Trace.Error(ex);
 
                     // don't retry if SkipSessionRecover = true, DT service will delete agent session to stop agent from taking more jobs.
-                    if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && await CreateSessionAsync(token))
+                    if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && (await CreateSessionAsync(token) == CreateSessionResult.Success))
                     {
                         Trace.Info($"{nameof(TaskAgentSessionExpiredException)} received, recovered by recreate session.");
                     }
@@ -311,16 +368,27 @@ namespace GitHub.Runner.Listener
 
                 if (message == null)
                 {
+                    continuousEmptyMessage++;
                     if (heartbeat.Elapsed > TimeSpan.FromMinutes(30))
                     {
                         Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes.");
                         heartbeat.Restart();
+                        continuousEmptyMessage = 0;
                     }
                     else
                     {
                         Trace.Verbose($"No message retrieved from session '{_session.SessionId}'.");
                     }
 
+                    if (continuousEmptyMessage > 50)
+                    {
+                        // retried more than 50 times in less than 30mins and still getting empty message
+                        // something is not right on the service side, backoff for 15-30s before retry
+                        _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval);
+                        Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
+                        await HostContext.Delay(_getNextMessageRetryInterval, token);
+                    }
+
                     continue;
                 }
 
@@ -343,6 +411,12 @@ namespace GitHub.Runner.Listener
             }
         }
 
+        public async Task RefreshListenerTokenAsync(CancellationToken cancellationToken)
+        {
+            await _runnerServer.RefreshConnectionAsync(RunnerConnectionType.MessageQueue, TimeSpan.FromSeconds(60));
+            await _brokerServer.ForceRefreshConnection(_creds);
+        }
+
         private TaskAgentMessage DecryptMessage(TaskAgentMessage message)
         {
             if (_session.EncryptionKey == null ||
@@ -392,6 +466,7 @@ namespace GitHub.Runner.Listener
                 ex is TaskAgentPoolNotFoundException ||
                 ex is TaskAgentSessionExpiredException ||
                 ex is AccessDeniedException ||
+                ex is RunnerNotFoundException ||
                 ex is VssUnauthorizedException)
             {
                 Trace.Info($"Non-retriable exception: {ex.Message}");
@@ -458,7 +533,8 @@ namespace GitHub.Runner.Listener
             }
             else if (ex is TaskAgentPoolNotFoundException ||
                      ex is AccessDeniedException ||
-                     ex is VssUnauthorizedException)
+                     ex is VssUnauthorizedException ||
+                     (ex is VssOAuthTokenRequestException oauthEx && oauthEx.Error != "server_error"))
             {
                 Trace.Info($"Non-retriable exception: {ex.Message}");
                 return false;

src/Runner.Listener/Program.cs

@@ -7,6 +7,7 @@ using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.WebApi;
+using GitHub.Services.WebApi;
 
 namespace GitHub.Runner.Listener
 {
@@ -144,6 +145,12 @@ namespace GitHub.Runner.Listener
                 trace.Error(e);
                 return Constants.Runner.ReturnCode.TerminatedError;
             }
+            catch (RunnerNotFoundException e)
+            {
+                terminal.WriteError($"An error occurred: {e.Message}");
+                trace.Error(e);
+                return Constants.Runner.ReturnCode.TerminatedError;
+            }
             catch (Exception e)
             {
                 terminal.WriteError($"An error occurred: {e.Message}");

src/Runner.Listener/Runner.Listener.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <OutputType>Exe</OutputType>
     <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
     <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
     <Version>$(Version)</Version>
     <PredefinedCulturesOnly>false</PredefinedCulturesOnly>
     <PublishReadyToRunComposite>true</PublishReadyToRunComposite>
@@ -18,11 +19,11 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
+    <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="System.IO.FileSystem.AccessControl" Version="4.4.0" />
-    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
-    <PackageReference Include="System.ServiceProcess.ServiceController" Version="4.4.0" />
+    <PackageReference Include="System.IO.FileSystem.AccessControl" Version="5.0.0" />
+    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
+    <PackageReference Include="System.ServiceProcess.ServiceController" Version="8.0.0" />
   </ItemGroup>
 
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">

src/Runner.Listener/Runner.cs

@@ -32,10 +32,25 @@ namespace GitHub.Runner.Listener
         private bool _inConfigStage;
         private ManualResetEvent _completedCommand = new(false);
 
+        // <summary>
+        // Helps avoid excessive calls to Run Service when encountering non-retriable errors from /acquirejob.
+        // Normally we rely on the HTTP clients to back off between retry attempts. However, acquiring a job
+        // involves calls to both Run Serivce and Broker. And Run Service and Broker communicate with each other
+        // in an async fashion.
+        //
+        // When Run Service encounters a non-retriable error, it sends an async message to Broker. The runner will,
+        // however, immediately call Broker to get the next message. If the async event from Run Service to Broker
+        // has not yet been processed, the next message from Broker may be the same job message.
+        //
+        // The error throttler helps us back off when encountering successive, non-retriable errors from /acquirejob.
+        // </summary>
+        private IErrorThrottler _acquireJobThrottler;
+
         public override void Initialize(IHostContext hostContext)
         {
             base.Initialize(hostContext);
             _term = HostContext.GetService<ITerminal>();
+            _acquireJobThrottler = HostContext.CreateService<IErrorThrottler>();
         }
 
         public async Task<int> ExecuteCommand(CommandSettings command)
@@ -213,15 +228,21 @@ namespace GitHub.Runner.Listener
                             var configFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), config.Key);
                             var configContent = Convert.FromBase64String(config.Value);
 #if OS_WINDOWS
+#pragma warning disable CA1416
                             if (configFile == HostContext.GetConfigFile(WellKnownConfigFile.RSACredentials))
                             {
                                 configContent = ProtectedData.Protect(configContent, null, DataProtectionScope.LocalMachine);
                             }
+#pragma warning restore CA1416
 #endif
                             File.WriteAllBytes(configFile, configContent);
                             File.SetAttributes(configFile, File.GetAttributes(configFile) | FileAttributes.Hidden);
                             Trace.Info($"Saved {configContent.Length} bytes to '{configFile}'.");
                         }
+
+                        // make sure we have the right user agent data added from the jitconfig
+                        HostContext.LoadDefaultUserAgents();
+                        VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy);
                     }
                     catch (Exception ex)
                     {
@@ -339,7 +360,7 @@ namespace GitHub.Runner.Listener
             }
         }
 
-        private IMessageListener GetMesageListener(RunnerSettings settings)
+        private IMessageListener GetMessageListener(RunnerSettings settings)
         {
             if (settings.UseV2Flow)
             {
@@ -358,8 +379,13 @@ namespace GitHub.Runner.Listener
             try
             {
                 Trace.Info(nameof(RunAsync));
-                _listener = GetMesageListener(settings);
-                if (!await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken))
+                _listener = GetMessageListener(settings);
+                CreateSessionResult createSessionResult = await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken);
+                if (createSessionResult == CreateSessionResult.SessionConflict)
+                {
+                    return Constants.Runner.ReturnCode.SessionConflict;
+                }
+                else if (createSessionResult == CreateSessionResult.Failure)
                 {
                     return Constants.Runner.ReturnCode.TerminatedError;
                 }
@@ -558,13 +584,21 @@ namespace GitHub.Runner.Listener
                                         await runServer.ConnectAsync(new Uri(messageRef.RunServiceUrl), creds);
                                         try
                                         {
-                                            jobRequestMessage =
-                                            await runServer.GetJobMessageAsync(messageRef.RunnerRequestId,
-                                            messageQueueLoopTokenSource.Token);
+                                            jobRequestMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageRef.BillingOwnerId, messageQueueLoopTokenSource.Token);
+                                            _acquireJobThrottler.Reset();
                                         }
-                                        catch (TaskOrchestrationJobAlreadyAcquiredException)
+                                        catch (Exception ex) when (
+                                            ex is TaskOrchestrationJobNotFoundException ||          // HTTP status 404
+                                            ex is TaskOrchestrationJobAlreadyAcquiredException ||   // HTTP status 409
+                                            ex is TaskOrchestrationJobUnprocessableException)       // HTTP status 422
                                         {
-                                            Trace.Info("Job is already acquired, skip this message.");
+                                            Trace.Info($"Skipping message Job. {ex.Message}");
+                                            await _acquireJobThrottler.IncrementAndWaitAsync(messageQueueLoopTokenSource.Token);
+                                            continue;
+                                        }
+                                        catch (Exception ex)
+                                        {
+                                            Trace.Error($"Caught exception from acquiring job message: {ex}");
                                             continue;
                                         }
                                     }
@@ -596,6 +630,22 @@ namespace GitHub.Runner.Listener
                                 Trace.Info($"Service requests the hosted runner to shutdown. Reason: '{HostedRunnerShutdownMessage.Reason}'.");
                                 return Constants.Runner.ReturnCode.Success;
                             }
+                            else if (string.Equals(message.MessageType, TaskAgentMessageTypes.ForceTokenRefresh))
+                            {
+                                Trace.Info("Received ForceTokenRefreshMessage");
+                                await _listener.RefreshListenerTokenAsync(messageQueueLoopTokenSource.Token);
+                            }
+                            else if (string.Equals(message.MessageType, RunnerRefreshConfigMessage.MessageType))
+                            {
+                                var runnerRefreshConfigMessage = JsonUtility.FromString<RunnerRefreshConfigMessage>(message.Body);
+                                Trace.Info($"Received RunnerRefreshConfigMessage for '{runnerRefreshConfigMessage.ConfigType}' config file");
+                                var configUpdater = HostContext.GetService<IRunnerConfigUpdater>();
+                                await configUpdater.UpdateRunnerConfigAsync(
+                                    runnerQualifiedId: runnerRefreshConfigMessage.RunnerQualifiedId,
+                                    configType: runnerRefreshConfigMessage.ConfigType,
+                                    serviceType: runnerRefreshConfigMessage.ServiceType,
+                                    configRefreshUrl: runnerRefreshConfigMessage.ConfigRefreshUrl);
+                            }
                             else
                             {
                                 Trace.Error($"Received message {message.MessageId} with unsupported message type {message.MessageType}.");
@@ -634,6 +684,7 @@ namespace GitHub.Runner.Listener
                     {
                         try
                         {
+                            Trace.Info("Deleting Runner Session...");
                             await _listener.DeleteSessionAsync();
                         }
                         catch (Exception ex) when (runOnce)
@@ -657,6 +708,10 @@ namespace GitHub.Runner.Listener
             {
                 Trace.Info("Runner OAuth token has been revoked. Shutting down.");
             }
+            catch (HostedRunnerDeprovisionedException)
+            {
+                Trace.Info("Hosted runner has been deprovisioned. Shutting down.");
+            }
 
             return Constants.Runner.ReturnCode.Success;
         }

src/Runner.Listener/RunnerConfigUpdater.cs

@@ -0,0 +1,267 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.Runner.Common;
+using GitHub.Runner.Sdk;
+using GitHub.Services.Common;
+
+namespace GitHub.Runner.Listener
+{
+    [ServiceLocator(Default = typeof(RunnerConfigUpdater))]
+    public interface IRunnerConfigUpdater : IRunnerService
+    {
+        Task UpdateRunnerConfigAsync(string runnerQualifiedId, string configType, string serviceType, string configRefreshUrl);
+    }
+
+    public sealed class RunnerConfigUpdater : RunnerService, IRunnerConfigUpdater
+    {
+        private RunnerSettings _settings;
+        private CredentialData _credData;
+        private IRunnerServer _runnerServer;
+        private IConfigurationStore _store;
+
+        public override void Initialize(IHostContext hostContext)
+        {
+            base.Initialize(hostContext);
+            _store = hostContext.GetService<IConfigurationStore>();
+            _settings = _store.GetSettings();
+            _credData = _store.GetCredentials();
+            _runnerServer = HostContext.GetService<IRunnerServer>();
+        }
+
+        public async Task UpdateRunnerConfigAsync(string runnerQualifiedId, string configType, string serviceType, string configRefreshUrl)
+        {
+            Trace.Entering();
+            try
+            {
+                ArgUtil.NotNullOrEmpty(runnerQualifiedId, nameof(runnerQualifiedId));
+                ArgUtil.NotNullOrEmpty(configType, nameof(configType));
+                ArgUtil.NotNullOrEmpty(serviceType, nameof(serviceType));
+                ArgUtil.NotNullOrEmpty(configRefreshUrl, nameof(configRefreshUrl));
+
+                // make sure the runner qualified id matches the current runner
+                if (!await VerifyRunnerQualifiedId(runnerQualifiedId))
+                {
+                    return;
+                }
+
+                // keep the timeout short to avoid blocking the main thread
+                using (var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
+                {
+                    switch (configType.ToLowerInvariant())
+                    {
+                        case "runner":
+                            await UpdateRunnerSettingsAsync(serviceType, configRefreshUrl, tokenSource.Token);
+                            break;
+                        case "credentials":
+                            await UpdateRunnerCredentialsAsync(serviceType, configRefreshUrl, tokenSource.Token);
+                            break;
+                        default:
+                            Trace.Error($"Invalid config type '{configType}'.");
+                            await ReportTelemetryAsync($"Invalid config type '{configType}'.");
+                            return;
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Trace.Error($"Failed to update runner '{configType}' config.");
+                Trace.Error(ex);
+                await ReportTelemetryAsync($"Failed to update runner '{configType}' config: {ex}");
+            }
+        }
+
+        private async Task UpdateRunnerSettingsAsync(string serviceType, string configRefreshUrl, CancellationToken token)
+        {
+            Trace.Entering();
+            // read the current runner settings and encode with base64
+            var runnerConfig = HostContext.GetConfigFile(WellKnownConfigFile.Runner);
+            string runnerConfigContent = File.ReadAllText(runnerConfig, Encoding.UTF8);
+            var encodedConfig = Convert.ToBase64String(Encoding.UTF8.GetBytes(runnerConfigContent));
+            if (string.IsNullOrEmpty(encodedConfig))
+            {
+                await ReportTelemetryAsync("Failed to get encoded runner settings.");
+                return;
+            }
+
+            // exchange the encoded runner settings with the service
+            string refreshedEncodedConfig = await RefreshRunnerConfigAsync(encodedConfig, serviceType, "runner", configRefreshUrl, token);
+            if (string.IsNullOrEmpty(refreshedEncodedConfig))
+            {
+                // service will return empty string if there is no change in the config
+                return;
+            }
+
+            var decodedConfig = Encoding.UTF8.GetString(Convert.FromBase64String(refreshedEncodedConfig));
+            RunnerSettings refreshedRunnerConfig;
+            try
+            {
+                refreshedRunnerConfig = StringUtil.ConvertFromJson<RunnerSettings>(decodedConfig);
+            }
+            catch (Exception ex)
+            {
+                Trace.Error($"Failed to convert runner config from json '{decodedConfig}'.");
+                Trace.Error(ex);
+                await ReportTelemetryAsync($"Failed to convert runner config '{decodedConfig}' from json: {ex}");
+                return;
+            }
+
+            // make sure the runner id and name in the refreshed config match the current runner
+            if (refreshedRunnerConfig?.AgentId != _settings.AgentId)
+            {
+                Trace.Error($"Runner id in refreshed config '{refreshedRunnerConfig?.AgentId.ToString() ?? "Empty"}' does not match the current runner '{_settings.AgentId}'.");
+                await ReportTelemetryAsync($"Runner id in refreshed config '{refreshedRunnerConfig?.AgentId.ToString() ?? "Empty"}' does not match the current runner '{_settings.AgentId}'.");
+                return;
+            }
+
+            if (refreshedRunnerConfig?.AgentName != _settings.AgentName)
+            {
+                Trace.Error($"Runner name in refreshed config '{refreshedRunnerConfig?.AgentName ?? "Empty"}' does not match the current runner '{_settings.AgentName}'.");
+                await ReportTelemetryAsync($"Runner name in refreshed config '{refreshedRunnerConfig?.AgentName ?? "Empty"}' does not match the current runner '{_settings.AgentName}'.");
+                return;
+            }
+
+            // save the refreshed runner settings as a separate file
+            _store.SaveMigratedSettings(refreshedRunnerConfig);
+            await ReportTelemetryAsync("Runner settings updated successfully.");
+        }
+
+        private async Task UpdateRunnerCredentialsAsync(string serviceType, string configRefreshUrl, CancellationToken token)
+        {
+            Trace.Entering();
+            // read the current runner credentials and encode with base64
+            var credConfig = HostContext.GetConfigFile(WellKnownConfigFile.Credentials);
+            string credConfigContent = File.ReadAllText(credConfig, Encoding.UTF8);
+            var encodedConfig = Convert.ToBase64String(Encoding.UTF8.GetBytes(credConfigContent));
+            if (string.IsNullOrEmpty(encodedConfig))
+            {
+                await ReportTelemetryAsync("Failed to get encoded credentials.");
+                return;
+            }
+
+            CredentialData currentCred = _store.GetCredentials();
+            if (currentCred == null)
+            {
+                await ReportTelemetryAsync("Failed to get current credentials.");
+                return;
+            }
+
+            // we only support refreshing OAuth credentials which is used by self-hosted runners.
+            if (currentCred.Scheme != Constants.Configuration.OAuth)
+            {
+                await ReportTelemetryAsync($"Not supported credential scheme '{currentCred.Scheme}'.");
+                return;
+            }
+
+            // exchange the encoded runner credentials with the service
+            string refreshedEncodedConfig = await RefreshRunnerConfigAsync(encodedConfig, serviceType, "credentials", configRefreshUrl, token);
+            if (string.IsNullOrEmpty(refreshedEncodedConfig))
+            {
+                // service will return empty string if there is no change in the config
+                return;
+            }
+
+            var decodedConfig = Encoding.UTF8.GetString(Convert.FromBase64String(refreshedEncodedConfig));
+            CredentialData refreshedCredConfig;
+            try
+            {
+                refreshedCredConfig = StringUtil.ConvertFromJson<CredentialData>(decodedConfig);
+            }
+            catch (Exception ex)
+            {
+                Trace.Error($"Failed to convert credentials config from json '{decodedConfig}'.");
+                Trace.Error(ex);
+                await ReportTelemetryAsync($"Failed to convert credentials config '{decodedConfig}' from json: {ex}");
+                return;
+            }
+
+            // make sure the credential scheme in the refreshed config match the current credential scheme
+            if (refreshedCredConfig?.Scheme != _credData.Scheme)
+            {
+                Trace.Error($"Credential scheme in refreshed config '{refreshedCredConfig?.Scheme ?? "Empty"}' does not match the current credential scheme '{_credData.Scheme}'.");
+                await ReportTelemetryAsync($"Credential scheme in refreshed config '{refreshedCredConfig?.Scheme ?? "Empty"}' does not match the current credential scheme '{_credData.Scheme}'.");
+                return;
+            }
+
+            if (_credData.Scheme == Constants.Configuration.OAuth)
+            {
+                // make sure the credential clientId in the refreshed config match the current credential clientId for OAuth auth scheme
+                var clientId = _credData.Data.GetValueOrDefault("clientId", null);
+                var refreshedClientId = refreshedCredConfig.Data.GetValueOrDefault("clientId", null);
+                if (clientId != refreshedClientId)
+                {
+                    Trace.Error($"Credential clientId in refreshed config '{refreshedClientId ?? "Empty"}' does not match the current credential clientId '{clientId}'.");
+                    await ReportTelemetryAsync($"Credential clientId in refreshed config '{refreshedClientId ?? "Empty"}' does not match the current credential clientId '{clientId}'.");
+                    return;
+                }
+            }
+
+            // save the refreshed runner credentials as a separate file
+            _store.SaveMigratedCredential(refreshedCredConfig);
+            await ReportTelemetryAsync("Runner credentials updated successfully.");
+        }
+
+        private async Task<bool> VerifyRunnerQualifiedId(string runnerQualifiedId)
+        {
+            Trace.Entering();
+            Trace.Info($"Verifying runner qualified id: {runnerQualifiedId}");
+            var idParts = runnerQualifiedId.Split("/", StringSplitOptions.RemoveEmptyEntries);
+            if (idParts.Length != 4 || idParts[3] != _settings.AgentId.ToString())
+            {
+                Trace.Error($"Runner qualified id '{runnerQualifiedId}' does not match the current runner '{_settings.AgentId}'.");
+                await ReportTelemetryAsync($"Runner qualified id '{runnerQualifiedId}' does not match the current runner '{_settings.AgentId}'.");
+                return false;
+            }
+            return true;
+        }
+
+        private async Task<string> RefreshRunnerConfigAsync(string encodedConfig, string serviceType, string configType, string configRefreshUrl, CancellationToken token)
+        {
+            string refreshedEncodedConfig;
+            switch (serviceType.ToLowerInvariant())
+            {
+                case "pipelines":
+                    try
+                    {
+                        refreshedEncodedConfig = await _runnerServer.RefreshRunnerConfigAsync((int)_settings.AgentId, configType, encodedConfig, token);
+                    }
+                    catch (Exception ex)
+                    {
+                        Trace.Error($"Failed to refresh runner {configType} config with service.");
+                        Trace.Error(ex);
+                        await ReportTelemetryAsync($"Failed to refresh {configType} config: {ex}");
+                        return null;
+                    }
+                    break;
+                case "runner-admin":
+                    throw new NotSupportedException("Runner admin service is not supported.");
+                default:
+                    Trace.Error($"Invalid service type '{serviceType}'.");
+                    await ReportTelemetryAsync($"Invalid service type '{serviceType}'.");
+                    return null;
+            }
+
+            return refreshedEncodedConfig;
+        }
+
+        private async Task ReportTelemetryAsync(string telemetry)
+        {
+            Trace.Entering();
+            try
+            {
+                using (var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
+                {
+                    await _runnerServer.UpdateAgentUpdateStateAsync(_settings.PoolId, _settings.AgentId, "RefreshConfig", telemetry, tokenSource.Token);
+                }
+            }
+            catch (Exception ex)
+            {
+                Trace.Error("Failed to report telemetry.");
+                Trace.Error(ex);
+            }
+        }
+    }
+}

src/Runner.Listener/RunnerJobRequestRef.cs

@@ -7,9 +7,14 @@ namespace GitHub.Runner.Listener
     {
         [DataMember(Name = "id")]
         public string Id { get; set; }
+        
         [DataMember(Name = "runner_request_id")]
         public string RunnerRequestId { get; set; }
+        
         [DataMember(Name = "run_service_url")]
         public string RunServiceUrl { get; set; }
+        
+        [DataMember(Name = "billing_owner_id")]
+        public string BillingOwnerId { get; set; }
     }
 }

src/Runner.Listener/SelfUpdater.cs

@@ -34,6 +34,7 @@ namespace GitHub.Runner.Listener
         private IRunnerServer _runnerServer;
         private int _poolId;
         private ulong _agentId;
+        private const int _numberOfOldVersionsToKeep = 1;
         private readonly ConcurrentQueue<string> _updateTrace = new();
         public bool Busy { get; private set; }
 
@@ -509,9 +510,9 @@ namespace GitHub.Runner.Listener
 
             // delete old bin.2.99.0 folder, only leave the current version and the latest download version
             var allBinDirs = Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "bin.*");
-            if (allBinDirs.Length > 2)
+            if (allBinDirs.Length > _numberOfOldVersionsToKeep)
             {
-                // there are more than 2 bin.version folder.
+                // there are more than one bin.version folder.
                 // delete older bin.version folders.
                 foreach (var oldBinDir in allBinDirs)
                 {
@@ -538,9 +539,9 @@ namespace GitHub.Runner.Listener
 
             // delete old externals.2.99.0 folder, only leave the current version and the latest download version
             var allExternalsDirs = Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "externals.*");
-            if (allExternalsDirs.Length > 2)
+            if (allExternalsDirs.Length > _numberOfOldVersionsToKeep)
             {
-                // there are more than 2 externals.version folder.
+                // there are more than one externals.version folder.
                 // delete older externals.version folders.
                 foreach (var oldExternalDir in allExternalsDirs)
                 {

src/Runner.PluginHost/Runner.PluginHost.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <OutputType>Exe</OutputType>
     <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
     <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
     <Version>$(Version)</Version>
     <PredefinedCulturesOnly>false</PredefinedCulturesOnly>
     <PublishReadyToRunComposite>true</PublishReadyToRunComposite>

src/Runner.Plugins/Runner.Plugins.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <OutputType>Library</OutputType>
     <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
     <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
     <Version>$(Version)</Version>
   </PropertyGroup>
 

src/Runner.Sdk/Runner.Sdk.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <OutputType>Library</OutputType>
     <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
     <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
     <Version>$(Version)</Version>
   </PropertyGroup>
 
@@ -14,9 +15,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="System.Text.Encoding.CodePages" Version="4.4.0" />
-    <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
-    <PackageReference Include="System.Threading.Channels" Version="4.4.0" />
+    <PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
+    <PackageReference Include="System.Threading.Channels" Version="8.0.0" />
   </ItemGroup>
 
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">

src/Runner.Sdk/Util/IOUtil.cs

@@ -459,6 +459,34 @@ namespace GitHub.Runner.Sdk
             File.WriteAllText(path, null);
         }
 
+        /// <summary>
+        /// Replaces invalid file name characters with '_'
+        /// </summary>
+        public static string ReplaceInvalidFileNameChars(string fileName)
+        {
+            var result = new StringBuilder();
+            var invalidChars = Path.GetInvalidFileNameChars();
+
+            var current = 0; // Current index
+            while (current < fileName?.Length)
+            {
+                var next = fileName.IndexOfAny(invalidChars, current);
+                if (next >= 0)
+                {
+                    result.Append(fileName.Substring(current, next - current));
+                    result.Append('_');
+                    current = next + 1;
+                }
+                else
+                {
+                    result.Append(fileName.Substring(current));
+                    break;
+                }
+            }
+
+            return result.ToString();
+        }
+
         /// <summary>
         /// Recursively enumerates a directory without following directory reparse points.
         /// </summary>

src/Runner.Sdk/Util/UrlUtil.cs

@@ -60,5 +60,15 @@ namespace GitHub.Runner.Sdk
             }
             return string.Empty;
         }
+
+        public static string GetVssRequestId(HttpResponseHeaders headers)
+        {
+            if (headers != null &&
+                headers.TryGetValues("x-vss-e2eid", out var headerValues))
+            {
+                return headerValues.FirstOrDefault();
+            }
+            return string.Empty;
+        }
     }
 }

src/Runner.Sdk/Util/VssUtil.cs

@@ -23,7 +23,13 @@ namespace GitHub.Runner.Sdk
 
             if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0)
             {
-                headerValues.AddRange(VssClientHttpRequestSettings.Default.UserAgent);
+                foreach (var headerVal in VssClientHttpRequestSettings.Default.UserAgent)
+                {
+                    if (!headerValues.Contains(headerVal))
+                    {
+                        headerValues.Add(headerVal);
+                    }
+                }
             }
 
             VssClientHttpRequestSettings.Default.UserAgent = headerValues;
@@ -33,6 +39,23 @@ namespace GitHub.Runner.Sdk
             {
                 VssClientHttpRequestSettings.Default.ServerCertificateValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
             }
+
+            var rawHeaderValues = new List<ProductInfoHeaderValue>();
+            rawHeaderValues.AddRange(additionalUserAgents);
+            rawHeaderValues.Add(new ProductInfoHeaderValue($"({StringUtil.SanitizeUserAgentHeader(RuntimeInformation.OSDescription)})"));
+
+            if (RawClientHttpRequestSettings.Default.UserAgent != null && RawClientHttpRequestSettings.Default.UserAgent.Count > 0)
+            {
+                foreach (var headerVal in RawClientHttpRequestSettings.Default.UserAgent)
+                {
+                    if (!rawHeaderValues.Contains(headerVal))
+                    {
+                        rawHeaderValues.Add(headerVal);
+                    }
+                }
+            }
+
+            RawClientHttpRequestSettings.Default.UserAgent = rawHeaderValues;
         }
 
         public static VssConnection CreateConnection(
@@ -62,11 +85,6 @@ namespace GitHub.Runner.Sdk
                 settings.SendTimeout = TimeSpan.FromSeconds(Math.Min(Math.Max(httpRequestTimeoutSeconds, 100), 1200));
             }
 
-            if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("USE_BROKER_FLOW")))
-            {
-                settings.AllowAutoRedirectForBroker = true;
-            }
-
             // Remove Invariant from the list of accepted languages.
             //
             // The constructor of VssHttpRequestSettings (base class of VssClientHttpRequestSettings) adds the current

src/Runner.Sdk/Util/WhichUtil.cs

@@ -7,129 +7,6 @@ namespace GitHub.Runner.Sdk
     public static class WhichUtil
     {
         public static string Which(string command, bool require = false, ITraceWriter trace = null, string prependPath = null)
-        {
-            ArgUtil.NotNullOrEmpty(command, nameof(command));
-            trace?.Info($"Which: '{command}'");
-            if (Path.IsPathFullyQualified(command) && File.Exists(command))
-            {
-                trace?.Info($"Fully qualified path: '{command}'");
-                return command;
-            }
-            string path = Environment.GetEnvironmentVariable(PathUtil.PathVariable);
-            if (string.IsNullOrEmpty(path))
-            {
-                trace?.Info("PATH environment variable not defined.");
-                path = path ?? string.Empty;
-            }
-            if (!string.IsNullOrEmpty(prependPath))
-            {
-                path = PathUtil.PrependPath(prependPath, path);
-            }
-
-            string[] pathSegments = path.Split(new Char[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries);
-            for (int i = 0; i < pathSegments.Length; i++)
-            {
-                pathSegments[i] = Environment.ExpandEnvironmentVariables(pathSegments[i]);
-            }
-
-            foreach (string pathSegment in pathSegments)
-            {
-                if (!string.IsNullOrEmpty(pathSegment) && Directory.Exists(pathSegment))
-                {
-                    string[] matches = null;
-#if OS_WINDOWS
-                    string pathExt = Environment.GetEnvironmentVariable("PATHEXT");
-                    if (string.IsNullOrEmpty(pathExt))
-                    {
-                        // XP's system default value for PATHEXT system variable
-                        pathExt = ".com;.exe;.bat;.cmd;.vbs;.vbe;.js;.jse;.wsf;.wsh";
-                    }
-
-                    string[] pathExtSegments = pathExt.Split(new string[] { ";" }, StringSplitOptions.RemoveEmptyEntries);
-
-                    // if command already has an extension.
-                    if (pathExtSegments.Any(ext => command.EndsWith(ext, StringComparison.OrdinalIgnoreCase)))
-                    {
-                        try
-                        {
-                            matches = Directory.GetFiles(pathSegment, command);
-                        }
-                        catch (UnauthorizedAccessException ex)
-                        {
-                            trace?.Info("Ignore UnauthorizedAccess exception during Which.");
-                            trace?.Verbose(ex.ToString());
-                        }
-
-                        if (matches != null && matches.Length > 0 && IsPathValid(matches.First(), trace))
-                        {
-                            trace?.Info($"Location: '{matches.First()}'");
-                            return matches.First();
-                        }
-                    }
-                    else
-                    {
-                        string searchPattern;
-                        searchPattern = StringUtil.Format($"{command}.*");
-                        try
-                        {
-                            matches = Directory.GetFiles(pathSegment, searchPattern);
-                        }
-                        catch (UnauthorizedAccessException ex)
-                        {
-                            trace?.Info("Ignore UnauthorizedAccess exception during Which.");
-                            trace?.Verbose(ex.ToString());
-                        }
-
-                        if (matches != null && matches.Length > 0)
-                        {
-                            // add extension.
-                            for (int i = 0; i < pathExtSegments.Length; i++)
-                            {
-                                string fullPath = Path.Combine(pathSegment, $"{command}{pathExtSegments[i]}");
-                                if (matches.Any(p => p.Equals(fullPath, StringComparison.OrdinalIgnoreCase)) && IsPathValid(fullPath, trace))
-                                {
-                                    trace?.Info($"Location: '{fullPath}'");
-                                    return fullPath;
-                                }
-                            }
-                        }
-                    }
-#else
-                    try
-                    {
-                        matches = Directory.GetFiles(pathSegment, command);
-                    }
-                    catch (UnauthorizedAccessException ex)
-                    {
-                        trace?.Info("Ignore UnauthorizedAccess exception during Which.");
-                        trace?.Verbose(ex.ToString());
-                    }
-
-                    if (matches != null && matches.Length > 0 && IsPathValid(matches.First(), trace))
-                    {
-                        trace?.Info($"Location: '{matches.First()}'");
-                        return matches.First();
-                    }
-#endif
-                }
-            }
-
-#if OS_WINDOWS
-            trace?.Info($"{command}: command not found. Make sure '{command}' is installed and its location included in the 'Path' environment variable.");
-#else
-            trace?.Info($"{command}: command not found. Make sure '{command}' is installed and its location included in the 'PATH' environment variable.");
-#endif
-            if (require)
-            {
-                throw new FileNotFoundException(
-                    message: $"{command}: command not found",
-                    fileName: command);
-            }
-
-            return null;
-        }
-
-        public static string Which2(string command, bool require = false, ITraceWriter trace = null, string prependPath = null)
         {
             ArgUtil.NotNullOrEmpty(command, nameof(command));
             trace?.Info($"Which2: '{command}'");

src/Runner.Worker/ActionManager.cs

@@ -483,10 +483,6 @@ namespace GitHub.Runner.Worker
                             {
                                 // Load stored Ids for later load actions
                                 compositeAction.Steps[i].Id = _cachedEmbeddedStepIds[action.Id][i];
-                                if (string.IsNullOrEmpty(executionContext.Global.Variables.Get("DistributedTask.EnableCompositeActions")) && compositeAction.Steps[i].Reference.Type != Pipelines.ActionSourceType.Script)
-                                {
-                                    throw new Exception("`uses:` keyword is not currently supported.");
-                                }
                             }
                         }
                         else
@@ -703,11 +699,12 @@ namespace GitHub.Runner.Worker
                 catch (Exception ex) when (!executionContext.CancellationToken.IsCancellationRequested) // Do not retry if the run is cancelled.
                 {
                     // UnresolvableActionDownloadInfoException is a 422 client error, don't retry
+                    // NonRetryableActionDownloadInfoException is an non-retryable exception from Actions
                     // Some possible cases are:
                     // * Repo is rate limited
                     // * Repo or tag doesn't exist, or isn't public
                     // * Policy validation failed
-                    if (attempt < 3 && !(ex is WebApi.UnresolvableActionDownloadInfoException))
+                    if (attempt < 3 && !(ex is WebApi.UnresolvableActionDownloadInfoException) && !(ex is WebApi.NonRetryableActionDownloadInfoException))
                     {
                         executionContext.Output($"Failed to resolve action download info. Error: {ex.Message}");
                         executionContext.Debug(ex.ToString());
@@ -778,7 +775,19 @@ namespace GitHub.Runner.Worker
                 // make sure we get a clean folder ready to use.
                 IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
                 Directory.CreateDirectory(destDirectory);
-                executionContext.Output($"Download action repository '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}' (SHA:{downloadInfo.ResolvedSha})");
+
+                if (downloadInfo.PackageDetails != null) 
+                {
+                    executionContext.Output($"##[group]Download immutable action package '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}'");
+                    executionContext.Output($"Version: {downloadInfo.PackageDetails.Version}");
+                    executionContext.Output($"Digest: {downloadInfo.PackageDetails.ManifestDigest}");
+                    executionContext.Output($"Source commit SHA: {downloadInfo.ResolvedSha}");
+                    executionContext.Output("##[endgroup]");
+                } 
+                else 
+                {
+                    executionContext.Output($"Download action repository '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}' (SHA:{downloadInfo.ResolvedSha})");
+                }
             }
 
             //download and extract action in a temp folder and rename it on success
@@ -796,43 +805,40 @@ namespace GitHub.Runner.Worker
             try
             {
                 var useActionArchiveCache = false;
-                if (executionContext.Global.Variables.GetBoolean("DistributedTask.UseActionArchiveCache") == true)
+                var hasActionArchiveCache = false;
+                var actionArchiveCacheDir = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory);
+                if (!string.IsNullOrEmpty(actionArchiveCacheDir) &&
+                    Directory.Exists(actionArchiveCacheDir))
                 {
-                    var hasActionArchiveCache = false;
-                    var actionArchiveCacheDir = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory);
-                    if (!string.IsNullOrEmpty(actionArchiveCacheDir) &&
-                        Directory.Exists(actionArchiveCacheDir))
-                    {
-                        hasActionArchiveCache = true;
-                        Trace.Info($"Check if action archive '{downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha}' already exists in cache directory '{actionArchiveCacheDir}'");
+                    hasActionArchiveCache = true;
+                    Trace.Info($"Check if action archive '{downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha}' already exists in cache directory '{actionArchiveCacheDir}'");
 #if OS_WINDOWS
-                        var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.zip");
+                    var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.zip");
 #else
-                        var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.tar.gz");
+                    var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.tar.gz");
 #endif
-                        if (File.Exists(cacheArchiveFile))
+                    if (File.Exists(cacheArchiveFile))
+                    {
+                        try
                         {
-                            try
-                            {
-                                Trace.Info($"Found action archive '{cacheArchiveFile}' in cache directory '{actionArchiveCacheDir}'");
-                                File.Copy(cacheArchiveFile, archiveFile);
-                                useActionArchiveCache = true;
-                                executionContext.Debug($"Copied action archive '{cacheArchiveFile}' to '{archiveFile}'");
-                            }
-                            catch (Exception ex)
-                            {
-                                Trace.Error($"Failed to copy action archive '{cacheArchiveFile}' to '{archiveFile}'. Error: {ex}");
-                            }
+                            Trace.Info($"Found action archive '{cacheArchiveFile}' in cache directory '{actionArchiveCacheDir}'");
+                            File.Copy(cacheArchiveFile, archiveFile);
+                            useActionArchiveCache = true;
+                            executionContext.Debug($"Copied action archive '{cacheArchiveFile}' to '{archiveFile}'");
+                        }
+                        catch (Exception ex)
+                        {
+                            Trace.Error($"Failed to copy action archive '{cacheArchiveFile}' to '{archiveFile}'. Error: {ex}");
                         }
                     }
-
-                    executionContext.Global.JobTelemetry.Add(new JobTelemetry()
-                    {
-                        Type = JobTelemetryType.General,
-                        Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache}"
-                    });
                 }
 
+                executionContext.Global.JobTelemetry.Add(new JobTelemetry()
+                {
+                    Type = JobTelemetryType.General,
+                    Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache}"
+                });
+
                 if (!useActionArchiveCache)
                 {
                     await DownloadRepositoryArchive(executionContext, link, downloadInfo.Authentication?.Token, archiveFile);
@@ -878,16 +884,9 @@ namespace GitHub.Runner.Worker
                     int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken);
                     if (exitCode != 0)
                     {
-                        if (executionContext.Global.Variables.GetBoolean("DistributedTask.DetailUntarFailure") == true)
-                        {
-                            var fileInfo = new FileInfo(archiveFile);
-                            var sha256hash = await IOUtil.GetFileContentSha256HashAsync(archiveFile);
-                            throw new InvalidActionArchiveException($"Can't use 'tar -xzf' extract archive file: {archiveFile} (SHA256 '{sha256hash}', size '{fileInfo.Length}' bytes, tar outputs '{string.Join(' ', tarOutputs)}'). Action being checked out: {downloadInfo.NameWithOwner}@{downloadInfo.Ref}. return code: {exitCode}.");
-                        }
-                        else
-                        {
-                            throw new InvalidActionArchiveException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. Action being checked out: {downloadInfo.NameWithOwner}@{downloadInfo.Ref}. return code: {exitCode}.");
-                        }
+                        var fileInfo = new FileInfo(archiveFile);
+                        var sha256hash = await IOUtil.GetFileContentSha256HashAsync(archiveFile);
+                        throw new InvalidActionArchiveException($"Can't use 'tar -xzf' extract archive file: {archiveFile} (SHA256 '{sha256hash}', size '{fileInfo.Length}' bytes, tar outputs '{string.Join(' ', tarOutputs)}'). Action being checked out: {downloadInfo.NameWithOwner}@{downloadInfo.Ref}. return code: {exitCode}.");
                     }
                 }
 #endif
@@ -1031,13 +1030,6 @@ namespace GitHub.Runner.Worker
                         }
                     }
 
-                    foreach (var step in compositeAction.Steps)
-                    {
-                        if (string.IsNullOrEmpty(executionContext.Global.Variables.Get("DistributedTask.EnableCompositeActions")) && step.Reference.Type != Pipelines.ActionSourceType.Script)
-                        {
-                            throw new Exception("`uses:` keyword is not currently supported.");
-                        }
-                    }
                     return setupInfo;
                 }
                 else
@@ -1122,6 +1114,7 @@ namespace GitHub.Runner.Worker
             int timeoutSeconds = 20 * 60;
             while (retryCount < 3)
             {
+                string requestId = string.Empty;
                 using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
                 using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken))
                 {
@@ -1137,7 +1130,7 @@ namespace GitHub.Runner.Worker
                             httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
                             using (var response = await httpClient.GetAsync(downloadUrl))
                             {
-                                var requestId = UrlUtil.GetGitHubRequestId(response.Headers);
+                                requestId = UrlUtil.GetGitHubRequestId(response.Headers);
                                 if (!string.IsNullOrEmpty(requestId))
                                 {
                                     Trace.Info($"Request URL: {downloadUrl} X-GitHub-Request-Id: {requestId} Http Status: {response.StatusCode}");
@@ -1175,7 +1168,7 @@ namespace GitHub.Runner.Worker
                     catch (OperationCanceledException ex) when (!executionContext.CancellationToken.IsCancellationRequested && retryCount >= 2)
                     {
                         Trace.Info($"Action download final retry timeout after {timeoutSeconds} seconds.");
-                        throw new TimeoutException($"Action '{downloadUrl}' download has timed out. Error: {ex.Message}");
+                        throw new TimeoutException($"Action '{downloadUrl}' download has timed out. Error: {ex.Message} {requestId}");
                     }
                     catch (ActionNotFoundException)
                     {
@@ -1190,11 +1183,11 @@ namespace GitHub.Runner.Worker
                         if (actionDownloadTimeout.Token.IsCancellationRequested)
                         {
                             // action download didn't finish within timeout
-                            executionContext.Warning($"Action '{downloadUrl}' didn't finish download within {timeoutSeconds} seconds.");
+                            executionContext.Warning($"Action '{downloadUrl}' didn't finish download within {timeoutSeconds} seconds. {requestId}");
                         }
                         else
                         {
-                            executionContext.Warning($"Failed to download action '{downloadUrl}'. Error: {ex.Message}");
+                            executionContext.Warning($"Failed to download action '{downloadUrl}'. Error: {ex.Message} {requestId}");
                         }
                     }
                 }

src/Runner.Worker/ActionManifestManager.cs

@@ -144,7 +144,7 @@ namespace GitHub.Runner.Worker
                     executionContext.Error(error.Message);
                 }
 
-                throw new ArgumentException($"Fail to load {fileRelativePath}");
+                throw new ArgumentException($"Failed to load {fileRelativePath}");
             }
 
             if (actionDefinition.Execution == null)

src/Runner.Worker/ContainerOperationProvider.cs

@@ -466,17 +466,39 @@ namespace GitHub.Runner.Worker
             {
                 throw new InvalidOperationException($"Failed to create directory to store registry client credentials: {e.Message}");
             }
-            var loginExitCode = await _dockerManager.DockerLogin(
-                executionContext,
-                configLocation,
-                container.RegistryServer,
-                container.RegistryAuthUsername,
-                container.RegistryAuthPassword);
 
-            if (loginExitCode != 0)
+            // Login docker with retry up to 3 times
+            int retryCount = 0;
+            int loginExitCode = 0;
+            while (retryCount < 3)
+            {
+                loginExitCode = await _dockerManager.DockerLogin(
+                    executionContext,
+                    configLocation,
+                    container.RegistryServer,
+                    container.RegistryAuthUsername,
+                    container.RegistryAuthPassword);
+                if (loginExitCode == 0)
+                {
+                    break;
+                }
+                else
+                {
+                    retryCount++;
+                    if (retryCount < 3)
+                    {
+                        var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
+                        executionContext.Warning($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
+                        await Task.Delay(backOff);
+                    }
+                }
+            }
+
+            if (retryCount == 3 && loginExitCode != 0)
             {
                 throw new InvalidOperationException($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}");
             }
+
             return configLocation;
         }
 

src/Runner.Worker/DiagnosticLogManager.cs

@@ -91,13 +91,13 @@ namespace GitHub.Runner.Worker
             string phaseName = executionContext.Global.Variables.System_PhaseDisplayName ?? "UnknownPhaseName";
 
             // zip the files
-            string diagnosticsZipFileName = $"{buildName}-{phaseName}.zip";
+            string diagnosticsZipFileName = $"{buildName}-{IOUtil.ReplaceInvalidFileNameChars(phaseName)}.zip";
             string diagnosticsZipFilePath = Path.Combine(supportRootFolder, diagnosticsZipFileName);
             ZipFile.CreateFromDirectory(supportFilesFolder, diagnosticsZipFilePath);
 
             // upload the json metadata file
             executionContext.Debug("Uploading diagnostic metadata file.");
-            string metadataFileName = $"diagnostics-{buildName}-{phaseName}.json";
+            string metadataFileName = $"diagnostics-{buildName}-{IOUtil.ReplaceInvalidFileNameChars(phaseName)}.json";
             string metadataFilePath = Path.Combine(supportFilesFolder, metadataFileName);
             string phaseResult = GetTaskResultAsString(executionContext.Result);
 
@@ -108,6 +108,8 @@ namespace GitHub.Runner.Worker
 
             parentContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath);
 
+            parentContext.QueueDiagnosticLogFile(name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath);
+
             executionContext.Debug("Diagnostic file upload complete.");
         }
 

src/Runner.Worker/ExecutionContext.cs

@@ -83,13 +83,14 @@ namespace GitHub.Runner.Worker
         // Initialize
         void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token);
         void CancelToken();
-        IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, ActionRunStage stage, Dictionary<string, string> intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool isEmbedded = false, CancellationTokenSource cancellationTokenSource = null, Guid embeddedId = default(Guid), string siblingScopeName = null, TimeSpan? timeout = null);
+        IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, ActionRunStage stage, Dictionary<string, string> intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool isEmbedded = false, List<Issue> embeddedIssueCollector = null, CancellationTokenSource cancellationTokenSource = null, Guid embeddedId = default(Guid), string siblingScopeName = null, TimeSpan? timeout = null);
         IExecutionContext CreateEmbeddedChild(string scopeName, string contextName, Guid embeddedId, ActionRunStage stage, Dictionary<string, string> intraActionState = null, string siblingScopeName = null);
 
         // logging
         long Write(string tag, string message);
         void QueueAttachFile(string type, string name, string filePath);
         void QueueSummaryFile(string name, string filePath, Guid stepRecordId);
+        void QueueDiagnosticLogFile(string name, string filePath);
 
         // timeline record update methods
         void Start(string currentOperation = null);
@@ -134,7 +135,6 @@ namespace GitHub.Runner.Worker
 
         private readonly TimelineRecord _record = new();
         private readonly Dictionary<Guid, TimelineRecord> _detailRecords = new();
-        private readonly List<Issue> _embeddedIssueCollector;
         private readonly object _loggerLock = new();
         private readonly object _matchersLock = new();
         private readonly ExecutionContext _parentExecutionContext;
@@ -153,6 +153,7 @@ namespace GitHub.Runner.Worker
         private CancellationTokenSource _cancellationTokenSource;
         private TaskCompletionSource<int> _forceCompleted = new();
         private bool _throttlingReported = false;
+        private List<Issue> _embeddedIssueCollector;
 
         // only job level ExecutionContext will track throttling delay.
         private long _totalThrottlingDelayInMilliseconds = 0;
@@ -355,6 +356,7 @@ namespace GitHub.Runner.Worker
             int? recordOrder = null,
             IPagingLogger logger = null,
             bool isEmbedded = false,
+            List<Issue> embeddedIssueCollector = null,
             CancellationTokenSource cancellationTokenSource = null,
             Guid embeddedId = default(Guid),
             string siblingScopeName = null,
@@ -364,6 +366,10 @@ namespace GitHub.Runner.Worker
 
             var child = new ExecutionContext(this, isEmbedded);
             child.Initialize(HostContext);
+            if ((Global.Variables.GetBoolean("RunService.FixEmbeddedIssues") ?? false) && embeddedIssueCollector != null)
+            {
+                child._embeddedIssueCollector = embeddedIssueCollector;
+            }
             child.Global = Global;
             child.ScopeName = scopeName;
             child.ContextName = contextName;
@@ -397,11 +403,11 @@ namespace GitHub.Runner.Worker
 
             if (recordOrder != null)
             {
-                child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, recordOrder);
+                child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, recordOrder, embedded: isEmbedded);
             }
             else
             {
-                child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, ++_childTimelineRecordOrder);
+                child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, ++_childTimelineRecordOrder, embedded: isEmbedded);
             }
             if (logger != null)
             {
@@ -432,7 +438,7 @@ namespace GitHub.Runner.Worker
             Dictionary<string, string> intraActionState = null,
             string siblingScopeName = null)
         {
-            return Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, contextName, stage, logger: _logger, isEmbedded: true, cancellationTokenSource: null, intraActionState: intraActionState, embeddedId: embeddedId, siblingScopeName: siblingScopeName, timeout: GetRemainingTimeout());
+            return Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, contextName, stage, logger: _logger, isEmbedded: true, embeddedIssueCollector: _embeddedIssueCollector, cancellationTokenSource: null, intraActionState: intraActionState, embeddedId: embeddedId, siblingScopeName: siblingScopeName, timeout: GetRemainingTimeout(), recordOrder: _record.Order);
         }
 
         public void Start(string currentOperation = null)
@@ -502,6 +508,9 @@ namespace GitHub.Runner.Worker
                     Status = _record.State,
                     Number = _record.Order,
                     Name = _record.Name,
+                    ActionName = StepTelemetry?.Action,
+                    Ref = StepTelemetry?.Ref,
+                    Type = StepTelemetry?.Type,
                     StartedAt = _record.StartTime,
                     CompletedAt = _record.FinishTime,
                     Annotations = new List<Annotation>()
@@ -519,7 +528,6 @@ namespace GitHub.Runner.Worker
                 Global.StepsResult.Add(stepResult);
             }
 
-
             if (Root != this)
             {
                 // only dispose TokenSource for step level ExecutionContext
@@ -807,11 +815,6 @@ namespace GitHub.Runner.Worker
 
             Global.Variables = new Variables(HostContext, variables);
 
-            if (Global.Variables.GetBoolean("DistributedTask.ForceInternalNodeVersionOnRunnerTo16") ?? false)
-            {
-                Environment.SetEnvironmentVariable(Constants.Variables.Agent.ForcedInternalNodeVersion, "node16");
-            }
-
             // Environment variables shared across all actions
             Global.EnvironmentVariables = new Dictionary<string, string>(VarUtil.EnvironmentVariableKeyComparer);
 
@@ -836,7 +839,6 @@ namespace GitHub.Runner.Worker
             // Actions environment
             ActionsEnvironment = message.ActionsEnvironment;
 
-
             // Service container info
             Global.ServiceContainers = new List<ContainerInfo>();
 
@@ -982,6 +984,18 @@ namespace GitHub.Runner.Worker
             _jobServerQueue.QueueResultsUpload(stepRecordId, name, filePath, ChecksAttachmentType.StepSummary, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0);
         }
 
+        public void QueueDiagnosticLogFile(string name, string filePath)
+        {
+            ArgUtil.NotNullOrEmpty(name, nameof(name));
+            ArgUtil.NotNullOrEmpty(filePath, nameof(filePath));
+
+            if (!File.Exists(filePath))
+            {
+                throw new FileNotFoundException($"Can't upload diagnostic log file: {filePath}. File does not exist.");
+            }
+            _jobServerQueue.QueueResultsUpload(_record.Id, name, filePath, CoreAttachmentType.ResultsDiagnosticLog, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0);
+        }
+
         // Add OnMatcherChanged
         public void Add(OnMatcherChanged handler)
         {
@@ -1160,7 +1174,7 @@ namespace GitHub.Runner.Worker
             }
         }
 
-        private void InitializeTimelineRecord(Guid timelineId, Guid timelineRecordId, Guid? parentTimelineRecordId, string recordType, string displayName, string refName, int? order)
+        private void InitializeTimelineRecord(Guid timelineId, Guid timelineRecordId, Guid? parentTimelineRecordId, string recordType, string displayName, string refName, int? order, bool embedded = false)
         {
             _mainTimelineId = timelineId;
             _record.Id = timelineRecordId;
@@ -1186,7 +1200,11 @@ namespace GitHub.Runner.Worker
             var configuration = HostContext.GetService<IConfigurationStore>();
             _record.WorkerName = configuration.GetSettings().AgentName;
 
-            _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
+            // We don't want to update the timeline record for embedded steps since they are not really represented in the UI.
+            if (!embedded)
+            {
+                _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
+            }
         }
 
         private void JobServerQueueThrottling_EventReceived(object sender, ThrottlingEventArgs data)
@@ -1401,7 +1419,7 @@ namespace GitHub.Runner.Worker
             {
                 if (key == PipelineTemplateConstants.HostWorkspace)
                 {
-                    // The HostWorkspace context var is excluded so that there is a var that always points to the host path. 
+                    // The HostWorkspace context var is excluded so that there is a var that always points to the host path.
                     // This var can be used to translate back from container paths, e.g. in HashFilesFunction, which always runs on the host machine
                     continue;
                 }

src/Runner.Worker/FileCommandManager.cs

@@ -244,7 +244,7 @@ namespace GitHub.Runner.Worker
                 if (resultsReceiverEndpoint != null)
                 {
                     Trace.Info($"Queueing results file ({filePath}) for attachment upload ({attachmentName})");
-                    var stepId = context.Id;
+                    var stepId = context.IsEmbedded ? context.EmbeddedId : context.Id;
                     // Attachments must be added to the parent context (job), not the current context (step)
                     context.Root.QueueSummaryFile(attachmentName, scrubbedFilePath, stepId);
                 }

src/Runner.Worker/Handlers/ContainerActionHandler.cs

@@ -223,6 +223,10 @@ namespace GitHub.Runner.Worker.Handlers
             {
                 Environment["ACTIONS_CACHE_URL"] = cacheUrl;
             }
+            if (systemConnection.Data.TryGetValue("PipelinesServiceUrl", out var pipelinesServiceUrl) && !string.IsNullOrEmpty(pipelinesServiceUrl))
+            {
+                Environment["ACTIONS_RUNTIME_URL"] = pipelinesServiceUrl;
+            }
             if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) && !string.IsNullOrEmpty(generateIdTokenUrl))
             {
                 Environment["ACTIONS_ID_TOKEN_REQUEST_URL"] = generateIdTokenUrl;

src/Runner.Worker/Handlers/HandlerFactory.cs

@@ -57,33 +57,13 @@ namespace GitHub.Runner.Worker.Handlers
                 handler = HostContext.CreateService<INodeScriptActionHandler>();
                 var nodeData = data as NodeJSActionExecutionData;
 
-                // With node12 EoL in 04/2022, we want to be able to uniformly upgrade all JS actions to node16 from the server
-                if (string.Equals(nodeData.NodeVersion, "node12", StringComparison.InvariantCultureIgnoreCase))
+                // With node12 EoL in 04/2022 and node16 EoL in 09/23, we want to execute all JS actions using node20
+                if (string.Equals(nodeData.NodeVersion, "node12", StringComparison.InvariantCultureIgnoreCase) ||
+                    string.Equals(nodeData.NodeVersion, "node16", StringComparison.InvariantCultureIgnoreCase))
                 {
-                    var repoAction = action as Pipelines.RepositoryPathReference;
-                    if (repoAction != null)
-                    {
-                        var warningActions = new HashSet<string>();
-                        if (executionContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode12DetectedAfterEndOfLifeEnvVariable, out var node16ForceWarnings))
-                        {
-                            warningActions = StringUtil.ConvertFromJson<HashSet<string>>(node16ForceWarnings);
-                        }
-
-                        string repoActionFullName;
-                        if (string.IsNullOrEmpty(repoAction.Name))
-                        {
-                            repoActionFullName = repoAction.Path; // local actions don't have a 'Name'
-                        }
-                        else
-                        {
-                            repoActionFullName = $"{repoAction.Name}/{repoAction.Path ?? string.Empty}".TrimEnd('/') + $"@{repoAction.Ref}";
-                        }
-
-                        warningActions.Add(repoActionFullName);
-                        executionContext.Global.Variables.Set("Node16ForceActionsWarnings", StringUtil.ConvertToJson(warningActions));
-                    }
-                    nodeData.NodeVersion = "node16";
+                    nodeData.NodeVersion = "node20";
                 }
+
                 (handler as INodeScriptActionHandler).Data = nodeData;
             }
             else if (data.ExecutionType == ActionExecutionType.Script)

src/Runner.Worker/Handlers/NodeScriptActionHandler.cs

@@ -58,6 +58,10 @@ namespace GitHub.Runner.Worker.Handlers
             {
                 Environment["ACTIONS_CACHE_URL"] = cacheUrl;
             }
+            if (systemConnection.Data.TryGetValue("PipelinesServiceUrl", out var pipelinesServiceUrl) && !string.IsNullOrEmpty(pipelinesServiceUrl))
+            {
+                Environment["ACTIONS_RUNTIME_URL"] = pipelinesServiceUrl;
+            }
             if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) && !string.IsNullOrEmpty(generateIdTokenUrl))
             {
                 Environment["ACTIONS_ID_TOKEN_REQUEST_URL"] = generateIdTokenUrl;
@@ -68,6 +72,11 @@ namespace GitHub.Runner.Worker.Handlers
                 Environment["ACTIONS_RESULTS_URL"] = resultsUrl;
             }
 
+            if (ExecutionContext.Global.Variables.GetBoolean("actions_uses_cache_service_v2") ?? false)
+            {
+                Environment["ACTIONS_CACHE_SERVICE_V2"] = bool.TrueString;
+            }
+
             // Resolve the target script.
             string target = null;
             if (stage == ActionRunStage.Main)
@@ -89,7 +98,6 @@ namespace GitHub.Runner.Worker.Handlers
                 ExecutionContext.StepTelemetry.HasPreStep = Data.HasPre;
                 ExecutionContext.StepTelemetry.HasPostStep = Data.HasPost;
             }
-            ExecutionContext.StepTelemetry.Type = Data.NodeVersion;
 
             ArgUtil.NotNullOrEmpty(target, nameof(target));
             target = Path.Combine(ActionDirectory, target);
@@ -102,19 +110,8 @@ namespace GitHub.Runner.Worker.Handlers
                 workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
             }
 
-            if (string.Equals(Data.NodeVersion, "node12", StringComparison.OrdinalIgnoreCase) &&
-                Constants.Runner.PlatformArchitecture.Equals(Constants.Architecture.Arm64))
-            {
-                ExecutionContext.Output($"The node12 is not supported. Use node16 instead.");
-                Data.NodeVersion = "node16";
-            }
-
-            string forcedNodeVersion = System.Environment.GetEnvironmentVariable(Constants.Variables.Agent.ForcedActionsNodeVersion);
-            if (forcedNodeVersion == "node16" && Data.NodeVersion != "node16")
-            {
-                Data.NodeVersion = "node16";
-            }
             var nodeRuntimeVersion = await StepHost.DetermineNodeRuntimeVersion(ExecutionContext, Data.NodeVersion);
+            ExecutionContext.StepTelemetry.Type = nodeRuntimeVersion;
             string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), nodeRuntimeVersion, "bin", $"node{IOUtil.ExeExtension}");
 
             // Format the arguments passed to node.
@@ -134,28 +131,6 @@ namespace GitHub.Runner.Worker.Handlers
             // Remove environment variable that may cause conflicts with the node within the runner.
             Environment.Remove("NODE_ICU_DATA"); // https://github.com/actions/runner/issues/795
 
-            if (string.Equals(Data.NodeVersion, Constants.Runner.DeprecatedNodeVersion, StringComparison.OrdinalIgnoreCase) && (ExecutionContext.Global.Variables.GetBoolean(Constants.Runner.Features.Node16Warning) ?? false))
-            {
-                var repoAction = Action as RepositoryPathReference;
-                var warningActions = new HashSet<string>();
-                if (ExecutionContext.Global.Variables.TryGetValue(Constants.Runner.DeprecatedNodeDetectedAfterEndOfLifeActions, out var deprecatedNodeWarnings))
-                {
-                    warningActions = StringUtil.ConvertFromJson<HashSet<string>>(deprecatedNodeWarnings);
-                }
-
-                if (string.IsNullOrEmpty(repoAction.Name))
-                {
-                    // local actions don't have a 'Name'
-                    warningActions.Add(repoAction.Path);
-                }
-                else
-                {
-                    warningActions.Add($"{repoAction.Name}/{repoAction.Path ?? string.Empty}".TrimEnd('/') + $"@{repoAction.Ref}");
-                }
-
-                ExecutionContext.Global.Variables.Set(Constants.Runner.DeprecatedNodeDetectedAfterEndOfLifeActions, StringUtil.ConvertToJson(warningActions));
-            }
-
             using (var stdoutManager = new OutputManager(ExecutionContext, ActionCommandManager))
             using (var stderrManager = new OutputManager(ExecutionContext, ActionCommandManager))
             {

src/Runner.Worker/Handlers/ScriptHandler.cs

@@ -83,40 +83,19 @@ namespace GitHub.Runner.Worker.Handlers
                 shellCommand = "pwsh";
                 if (validateShellOnHost)
                 {
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        shellCommandPath = WhichUtil.Which2(shellCommand, require: false, Trace, prependPath);
-                    }
-                    else
-                    {
-                        shellCommandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
-                    }
+                    shellCommandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
                     if (string.IsNullOrEmpty(shellCommandPath))
                     {
                         shellCommand = "powershell";
-                        Trace.Info($"Defaulting to {shellCommand}");
-                        if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                        {
-                            shellCommandPath = WhichUtil.Which2(shellCommand, require: true, Trace, prependPath);
-                        }
-                        else
-                        {
-                            shellCommandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath);
-                        }
+                        Trace.Info($"Defaulting to {shellCommand}");                        
+                        shellCommandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath);
                     }
                 }
 #else
                 shellCommand = "sh";
                 if (validateShellOnHost)
                 {
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        shellCommandPath = WhichUtil.Which2("bash", false, Trace, prependPath) ?? WhichUtil.Which2("sh", true, Trace, prependPath);
-                    }
-                    else
-                    {
-                        shellCommandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath);
-                    }
+                    shellCommandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath);
                 }
 #endif
                 argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand);
@@ -127,14 +106,7 @@ namespace GitHub.Runner.Worker.Handlers
                 shellCommand = parsed.shellCommand;
                 if (validateShellOnHost)
                 {
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        shellCommandPath = WhichUtil.Which2(parsed.shellCommand, true, Trace, prependPath);
-                    }
-                    else
-                    {
-                        shellCommandPath = WhichUtil.Which(parsed.shellCommand, true, Trace, prependPath);
-                    }
+                    shellCommandPath = WhichUtil.Which(parsed.shellCommand, true, Trace, prependPath);
                 }
 
                 argFormat = $"{parsed.shellArgs}".TrimStart();
@@ -216,38 +188,17 @@ namespace GitHub.Runner.Worker.Handlers
             {
 #if OS_WINDOWS
                 shellCommand = "pwsh";
-                if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                {
-                    commandPath = WhichUtil.Which2(shellCommand, require: false, Trace, prependPath);
-                }
-                else
-                {
-                    commandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
-                }
+                commandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
                 if (string.IsNullOrEmpty(commandPath))
                 {
                     shellCommand = "powershell";
                     Trace.Info($"Defaulting to {shellCommand}");
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        commandPath = WhichUtil.Which2(shellCommand, require: true, Trace, prependPath);
-                    }
-                    else
-                    {
-                        commandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath);
-                    }
+                    commandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath);
                 }
                 ArgUtil.NotNullOrEmpty(commandPath, "Default Shell");
 #else
                 shellCommand = "sh";
-                if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                {
-                    commandPath = WhichUtil.Which2("bash", false, Trace, prependPath) ?? WhichUtil.Which2("sh", true, Trace, prependPath);
-                }
-                else
-                {
-                    commandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath);
-                }
+                commandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath);
 #endif
                 argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand);
             }
@@ -258,14 +209,7 @@ namespace GitHub.Runner.Worker.Handlers
                 if (!IsActionStep && systemShells.Contains(shell))
                 {
                     shellCommand = shell;
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        commandPath = WhichUtil.Which2(shell, !isContainerStepHost, Trace, prependPath);
-                    }
-                    else
-                    {
-                        commandPath = WhichUtil.Which(shell, !isContainerStepHost, Trace, prependPath);
-                    }
+                    commandPath = WhichUtil.Which(shell, !isContainerStepHost, Trace, prependPath);
                     if (shell == "bash")
                     {
                         argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat("sh");
@@ -280,14 +224,7 @@ namespace GitHub.Runner.Worker.Handlers
                     var parsed = ScriptHandlerHelpers.ParseShellOptionString(shell);
                     shellCommand = parsed.shellCommand;
                     // For non-ContainerStepHost, the command must be located on the host by Which
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        commandPath = WhichUtil.Which2(parsed.shellCommand, !isContainerStepHost, Trace, prependPath);
-                    }
-                    else
-                    {
-                        commandPath = WhichUtil.Which(parsed.shellCommand, !isContainerStepHost, Trace, prependPath);
-                    }
+                    commandPath = WhichUtil.Which(parsed.shellCommand, !isContainerStepHost, Trace, prependPath);
                     argFormat = $"{parsed.shellArgs}".TrimStart();
                     if (string.IsNullOrEmpty(argFormat))
                     {

src/Runner.Worker/JobExtension.cs

@@ -17,6 +17,7 @@ using GitHub.Runner.Common;
 using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
+using Newtonsoft.Json;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
 namespace GitHub.Runner.Worker
@@ -42,11 +43,13 @@ namespace GitHub.Runner.Worker
     public sealed class JobExtension : RunnerService, IJobExtension
     {
         private readonly HashSet<string> _existingProcesses = new(StringComparer.OrdinalIgnoreCase);
-        private readonly List<Task<string>> _connectivityCheckTasks = new();
+        private readonly List<Task<CheckResult>> _connectivityCheckTasks = new();
         private bool _processCleanup;
         private string _processLookupId = $"github_{Guid.NewGuid()}";
         private CancellationTokenSource _diskSpaceCheckToken = new();
         private Task _diskSpaceCheckTask = null;
+        private CancellationTokenSource _serviceConnectivityCheckToken = new();
+        private Task _serviceConnectivityCheckTask = null;
 
         // Download all required actions.
         // Make sure all condition inputs are valid.
@@ -392,6 +395,18 @@ namespace GitHub.Runner.Worker
                         }
                     }
 
+                    // Register custom image creation post-job step if the "snapshot" token is present in the message.
+                    var snapshotRequest = templateEvaluator.EvaluateJobSnapshotRequest(message.Snapshot, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
+                    if (snapshotRequest != null)
+                    {
+                        var snapshotOperationProvider = HostContext.GetService<ISnapshotOperationProvider>();
+                        jobContext.RegisterPostJobStep(new JobExtensionRunner(
+                            runAsync: (executionContext, _) => snapshotOperationProvider.CreateSnapshotRequestAsync(executionContext, snapshotRequest),
+                            condition: snapshotRequest.Condition,
+                            displayName: $"Create custom image",
+                            data: null));
+                    }
+
                     // Register Job Completed hook if the variable is set
                     var completedHookPath = Environment.GetEnvironmentVariable("ACTIONS_RUNNER_HOOK_JOB_COMPLETED");
                     if (!string.IsNullOrEmpty(completedHookPath))
@@ -442,11 +457,14 @@ namespace GitHub.Runner.Worker
                         {
                             foreach (var checkUrl in checkUrls)
                             {
-                                _connectivityCheckTasks.Add(CheckConnectivity(checkUrl));
+                                _connectivityCheckTasks.Add(CheckConnectivity(checkUrl, accessToken: string.Empty, timeoutInSeconds: 5, token: CancellationToken.None));
                             }
                         }
                     }
 
+                    Trace.Info($"Start checking service connectivity in background.");
+                    _serviceConnectivityCheckTask = CheckServiceConnectivityAsync(context, _serviceConnectivityCheckToken.Token);
+
                     return steps;
                 }
                 catch (OperationCanceledException ex) when (jobContext.CancellationToken.IsCancellationRequested)
@@ -680,7 +698,7 @@ namespace GitHub.Runner.Worker
                             {
                                 var result = await check;
                                 Trace.Info($"Connectivity check result: {result}");
-                                context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = result });
+                                context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = $"{result.EndpointUrl}: {result.StatusCode}" });
                             }
                         }
                         catch (Exception ex)
@@ -690,6 +708,22 @@ namespace GitHub.Runner.Worker
                             context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = $"Fail to check server connectivity. {ex.Message}" });
                         }
                     }
+
+                    // Collect service connectivity check result
+                    if (_serviceConnectivityCheckTask != null)
+                    {
+                        _serviceConnectivityCheckToken.Cancel();
+                        try
+                        {
+                            await _serviceConnectivityCheckTask;
+                        }
+                        catch (Exception ex)
+                        {
+                            Trace.Error($"Fail to check service connectivity.");
+                            Trace.Error(ex);
+                            context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = $"Fail to check service connectivity. {ex.Message}" });
+                        }
+                    }
                 }
                 catch (Exception ex)
                 {
@@ -705,11 +739,13 @@ namespace GitHub.Runner.Worker
             }
         }
 
-        private async Task<string> CheckConnectivity(string endpointUrl)
+        private async Task<CheckResult> CheckConnectivity(string endpointUrl, string accessToken, int timeoutInSeconds, CancellationToken token)
         {
             Trace.Info($"Check server connectivity for {endpointUrl}.");
-            string result = string.Empty;
-            using (var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)))
+            CheckResult result = new CheckResult() { EndpointUrl = endpointUrl };
+            var stopwatch = Stopwatch.StartNew();
+            using (var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutInSeconds)))
+            using (var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutTokenSource.Token))
             {
                 try
                 {
@@ -717,21 +753,44 @@ namespace GitHub.Runner.Worker
                     using (var httpClient = new HttpClient(httpClientHandler))
                     {
                         httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
-                        var response = await httpClient.GetAsync(endpointUrl, timeoutTokenSource.Token);
-                        result = $"{endpointUrl}: {response.StatusCode}";
+                        if (!string.IsNullOrEmpty(accessToken))
+                        {
+                            httpClient.DefaultRequestHeaders.Add("Authorization", $"Bearer {accessToken}");
+                        }
+
+                        var response = await httpClient.GetAsync(endpointUrl, linkedTokenSource.Token);
+                        result.StatusCode = $"{response.StatusCode}";
+
+                        var githubRequestId = UrlUtil.GetGitHubRequestId(response.Headers);
+                        var vssRequestId = UrlUtil.GetVssRequestId(response.Headers);
+                        if (!string.IsNullOrEmpty(githubRequestId))
+                        {
+                            result.RequestId = githubRequestId;
+                        }
+                        else if (!string.IsNullOrEmpty(vssRequestId))
+                        {
+                            result.RequestId = vssRequestId;
+                        }
                     }
                 }
+                catch (Exception ex) when (ex is OperationCanceledException && token.IsCancellationRequested)
+                {
+                    Trace.Error($"Request canceled during connectivity check: {ex}");
+                    result.StatusCode = "canceled";
+                }
                 catch (Exception ex) when (ex is OperationCanceledException && timeoutTokenSource.IsCancellationRequested)
                 {
                     Trace.Error($"Request timeout during connectivity check: {ex}");
-                    result = $"{endpointUrl}: timeout";
+                    result.StatusCode = "timeout";
                 }
                 catch (Exception ex)
                 {
                     Trace.Error($"Catch exception during connectivity check: {ex}");
-                    result = $"{endpointUrl}: {ex.Message}";
+                    result.StatusCode = $"{ex.Message}";
                 }
             }
+            stopwatch.Stop();
+            result.DurationInMs = (int)stopwatch.ElapsedMilliseconds;
 
             return result;
         }
@@ -769,6 +828,84 @@ namespace GitHub.Runner.Worker
             }
         }
 
+        private async Task CheckServiceConnectivityAsync(IExecutionContext context, CancellationToken token)
+        {
+            var connectionTest = context.Global.Variables.Get(WellKnownDistributedTaskVariables.RunnerServiceConnectivityTest);
+            if (string.IsNullOrEmpty(connectionTest))
+            {
+                return;
+            }
+
+            ServiceConnectivityCheckInput checkConnectivityInfo;
+            try
+            {
+                checkConnectivityInfo = StringUtil.ConvertFromJson<ServiceConnectivityCheckInput>(connectionTest);
+            }
+            catch (Exception ex)
+            {
+                context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $"Fail to parse JSON. {ex.Message}" });
+                return;
+            }
+
+            if (checkConnectivityInfo == null)
+            {
+                return;
+            }
+
+            // make sure interval is at least 10 seconds
+            checkConnectivityInfo.IntervalInSecond = Math.Max(10, checkConnectivityInfo.IntervalInSecond);
+
+            var systemConnection = context.Global.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
+            var accessToken = systemConnection.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken];
+
+            var testResult = new ServiceConnectivityCheckResult();
+            while (!token.IsCancellationRequested)
+            {
+                foreach (var endpoint in checkConnectivityInfo.Endpoints)
+                {
+                    if (string.IsNullOrEmpty(endpoint.Key) || string.IsNullOrEmpty(endpoint.Value))
+                    {
+                        continue;
+                    }
+
+                    if (!testResult.EndpointsResult.ContainsKey(endpoint.Key))
+                    {
+                        testResult.EndpointsResult[endpoint.Key] = new List<string>();
+                    }
+
+                    try
+                    {
+                        var result = await CheckConnectivity(endpoint.Value, accessToken: accessToken, timeoutInSeconds: checkConnectivityInfo.RequestTimeoutInSecond, token);
+                        testResult.EndpointsResult[endpoint.Key].Add($"{result.StartTime:s}: {result.StatusCode} - {result.RequestId} - {result.DurationInMs}ms");
+                        if (!testResult.HasFailure &&
+                            result.StatusCode != "OK" &&
+                            result.StatusCode != "canceled")
+                        {
+                            // track if any endpoint is not reachable
+                            testResult.HasFailure = true;
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        testResult.EndpointsResult[endpoint.Key].Add($"{DateTime.UtcNow:s}: {ex.Message}");
+                    }
+                }
+
+                try
+                {
+                    await Task.Delay(TimeSpan.FromSeconds(checkConnectivityInfo.IntervalInSecond), token);
+                }
+                catch (TaskCanceledException)
+                {
+                    // ignore
+                }
+            }
+
+            var telemetryData = StringUtil.ConvertToJson(testResult, Formatting.None);
+            Trace.Verbose($"Connectivity check result: {telemetryData}");
+            context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = telemetryData });
+        }
+
         private Dictionary<int, Process> SnapshotProcesses()
         {
             Dictionary<int, Process> snapshot = new();
@@ -800,5 +937,23 @@ namespace GitHub.Runner.Worker
                 throw new ArgumentException("Jobs without a job container are forbidden on this runner, please add a 'container:' to your job or contact your self-hosted runner administrator.");
             }
         }
+
+        private class CheckResult
+        {
+            public CheckResult()
+            {
+                StartTime = DateTime.UtcNow;
+            }
+
+            public string EndpointUrl { get; set; }
+
+            public DateTime StartTime { get; set; }
+
+            public string StatusCode { get; set; }
+
+            public string RequestId { get; set; }
+
+            public int DurationInMs { get; set; }
+        }
     }
 }

src/Runner.Worker/JobRunner.cs

@@ -15,6 +15,7 @@ using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
 using GitHub.Services.WebApi;
+using Sdk.RSWebApi.Contracts;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
 namespace GitHub.Runner.Worker
@@ -42,20 +43,18 @@ namespace GitHub.Runner.Worker
             Trace.Info("Job ID {0}", message.JobId);
 
             DateTime jobStartTimeUtc = DateTime.UtcNow;
+            _runnerSettings = HostContext.GetService<IConfigurationStore>().GetSettings();
             IRunnerService server = null;
 
             // add orchestration id to useragent for better correlation.
             if (message.Variables.TryGetValue(Constants.Variables.System.OrchestrationId, out VariableValue orchestrationId) &&
                 !string.IsNullOrEmpty(orchestrationId.Value))
             {
-                HostContext.UserAgents.Add(new ProductInfoHeaderValue("OrchestrationId", orchestrationId.Value));
-            }
+                // make the orchestration id the first item in the user-agent header to avoid get truncated in server log.
+                HostContext.UserAgents.Insert(0, new ProductInfoHeaderValue("OrchestrationId", orchestrationId.Value));
 
-            var jobServerQueueTelemetry = false;
-            if (message.Variables.TryGetValue("DistributedTask.EnableJobServerQueueTelemetry", out VariableValue enableJobServerQueueTelemetry) &&
-                !string.IsNullOrEmpty(enableJobServerQueueTelemetry?.Value))
-            {
-                jobServerQueueTelemetry = StringUtil.ConvertToBoolean(enableJobServerQueueTelemetry.Value);
+                // make sure orchestration id is in the user-agent header.
+                VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy);
             }
 
             ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
@@ -79,7 +78,7 @@ namespace GitHub.Runner.Worker
                     launchServer.InitializeLaunchClient(new Uri(launchReceiverEndpoint), accessToken);
                 }
                 _jobServerQueue = HostContext.GetService<IJobServerQueue>();
-                _jobServerQueue.Start(message, resultsServiceOnly: true, enableTelemetry: jobServerQueueTelemetry);
+                _jobServerQueue.Start(message, resultsServiceOnly: true);
             }
             else
             {
@@ -101,7 +100,7 @@ namespace GitHub.Runner.Worker
                 VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, delegatingHandlers);
                 await jobServer.ConnectAsync(jobConnection);
 
-                _jobServerQueue.Start(message, enableTelemetry: jobServerQueueTelemetry);
+                _jobServerQueue.Start(message);
                 server = jobServer;
             }
 
@@ -161,8 +160,6 @@ namespace GitHub.Runner.Worker
 
                 jobContext.SetRunnerContext("os", VarUtil.OS);
                 jobContext.SetRunnerContext("arch", VarUtil.OSArchitecture);
-
-                _runnerSettings = HostContext.GetService<IConfigurationStore>().GetSettings();
                 jobContext.SetRunnerContext("name", _runnerSettings.AgentName);
 
                 if (jobContext.Global.Variables.TryGetValue(WellKnownDistributedTaskVariables.RunnerEnvironment, out var runnerEnvironment))
@@ -283,16 +280,12 @@ namespace GitHub.Runner.Worker
         {
             jobContext.Debug($"Finishing: {message.JobDisplayName}");
             TaskResult result = jobContext.Complete(taskResult);
-            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.DeprecatedNodeDetectedAfterEndOfLifeActions, out var deprecatedNodeWarnings))
-            {
-                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(deprecatedNodeWarnings));
-                jobContext.Warning(string.Format(Constants.Runner.DetectedNodeAfterEndOfLifeMessage, actions));
-            }
 
-            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode12DetectedAfterEndOfLifeEnvVariable, out var node16ForceWarnings))
+            var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: false);
+            // include any job telemetry from the background upload process.
+            if (jobQueueTelemetry?.Count > 0)
             {
-                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(node16ForceWarnings));
-                jobContext.Warning(string.Format(Constants.Runner.EnforcedNode12DetectedAfterEndOfLife, actions));
+                jobContext.Global.JobTelemetry.AddRange(jobQueueTelemetry);
             }
 
             // Make sure to clean temp after file upload since they may be pending fileupload still use the TEMP dir.
@@ -311,6 +304,13 @@ namespace GitHub.Runner.Worker
                 environmentUrl = urlStringToken.Value;
             }
 
+            // Get telemetry
+            IList<Telemetry> telemetry = null;
+            if (jobContext.Global.JobTelemetry.Count > 0)
+            {
+                telemetry = jobContext.Global.JobTelemetry.Select(x => new Telemetry { Type = x.Type.ToString(), Message = x.Message, }).ToList();
+            }
+
             Trace.Info($"Raising job completed against run service");
             var completeJobRetryLimit = 5;
             var exceptions = new List<Exception>();
@@ -318,9 +318,30 @@ namespace GitHub.Runner.Worker
             {
                 try
                 {
-                    await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, default);
+                    if (jobContext.Global.Variables.GetBoolean(Constants.Runner.Features.SkipRetryCompleteJobUponKnownErrors) ?? false)
+                    {
+                        await runServer.CompleteJob2Async(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, telemetry, billingOwnerId: message.BillingOwnerId, default);
+                    }
+                    else
+                    {
+                        await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, telemetry, billingOwnerId: message.BillingOwnerId, default);
+                    }
                     return result;
                 }
+                catch (VssUnauthorizedException ex) when (jobContext.Global.Variables.GetBoolean(Constants.Runner.Features.SkipRetryCompleteJobUponKnownErrors) ?? false)
+                {
+                    Trace.Error($"Catch exception while attempting to complete job {message.JobId}, job request {message.RequestId}.");
+                    Trace.Error(ex);
+                    exceptions.Add(ex);
+                    break;
+                }
+                catch (TaskOrchestrationJobNotFoundException ex) when (jobContext.Global.Variables.GetBoolean(Constants.Runner.Features.SkipRetryCompleteJobUponKnownErrors) ?? false)
+                {
+                    Trace.Error($"Catch exception while attempting to complete job {message.JobId}, job request {message.RequestId}.");
+                    Trace.Error(ex);
+                    exceptions.Add(ex);
+                    break;
+                }
                 catch (Exception ex)
                 {
                     Trace.Error($"Catch exception while attempting to complete job {message.JobId}, job request {message.RequestId}.");
@@ -343,68 +364,14 @@ namespace GitHub.Runner.Worker
 
             if (_runnerSettings.DisableUpdate == true)
             {
-                try
-                {
-                    var currentVersion = new PackageVersion(BuildConstants.RunnerPackage.Version);
-                    ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
-                    VssCredentials serverCredential = VssUtil.GetVssCredential(systemConnection);
-
-                    var runnerServer = HostContext.GetService<IRunnerServer>();
-                    await runnerServer.ConnectAsync(systemConnection.Url, serverCredential);
-                    var serverPackages = await runnerServer.GetPackagesAsync("agent", BuildConstants.RunnerPackage.PackageName, 5, includeToken: false, cancellationToken: CancellationToken.None);
-                    if (serverPackages.Count > 0)
-                    {
-                        serverPackages = serverPackages.OrderByDescending(x => x.Version).ToList();
-                        Trace.Info($"Newer packages {StringUtil.ConvertToJson(serverPackages.Select(x => x.Version.ToString()))}");
-
-                        var warnOnFailedJob = false; // any minor/patch version behind.
-                        var warnOnOldRunnerVersion = false; // >= 2 minor version behind
-                        if (serverPackages.Any(x => x.Version.CompareTo(currentVersion) > 0))
-                        {
-                            Trace.Info($"Current runner version {currentVersion} is behind the latest runner version {serverPackages[0].Version}.");
-                            warnOnFailedJob = true;
-                        }
-
-                        if (serverPackages.Where(x => x.Version.Major == currentVersion.Major && x.Version.Minor > currentVersion.Minor).Count() > 1)
-                        {
-                            Trace.Info($"Current runner version {currentVersion} is way behind the latest runner version {serverPackages[0].Version}.");
-                            warnOnOldRunnerVersion = true;
-                        }
-
-                        if (result == TaskResult.Failed && warnOnFailedJob)
-                        {
-                            jobContext.Warning($"This job failure may be caused by using an out of date self-hosted runner. You are currently using runner version {currentVersion}. Please update to the latest version {serverPackages[0].Version}");
-                        }
-                        else if (warnOnOldRunnerVersion)
-                        {
-                            jobContext.Warning($"This self-hosted runner is currently using runner version {currentVersion}. This version is out of date. Please update to the latest version {serverPackages[0].Version}");
-                        }
-                    }
-                }
-                catch (Exception ex)
-                {
-                    // Ignore any error since suggest runner update is best effort.
-                    Trace.Error($"Caught exception during runner version check: {ex}");
-                }
-            }
-
-            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.DeprecatedNodeDetectedAfterEndOfLifeActions, out var deprecatedNodeWarnings))
-            {
-                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(deprecatedNodeWarnings));
-                jobContext.Warning(string.Format(Constants.Runner.DetectedNodeAfterEndOfLifeMessage, actions));
-            }
-
-            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode12DetectedAfterEndOfLifeEnvVariable, out var node16ForceWarnings))
-            {
-                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(node16ForceWarnings));
-                jobContext.Warning(string.Format(Constants.Runner.EnforcedNode12DetectedAfterEndOfLife, actions));
+                await WarningOutdatedRunnerAsync(jobContext, message, result);
             }
 
             try
             {
                 var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: true);
                 // include any job telemetry from the background upload process.
-                if (jobQueueTelemetry.Count > 0)
+                if (jobQueueTelemetry?.Count > 0)
                 {
                     jobContext.Global.JobTelemetry.AddRange(jobQueueTelemetry);
                 }
@@ -532,5 +499,52 @@ namespace GitHub.Runner.Worker
 
             return Array.Empty<JobTelemetry>();
         }
+
+        private async Task WarningOutdatedRunnerAsync(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult result)
+        {
+            try
+            {
+                var currentVersion = new PackageVersion(BuildConstants.RunnerPackage.Version);
+                ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
+                VssCredentials serverCredential = VssUtil.GetVssCredential(systemConnection);
+
+                var runnerServer = HostContext.GetService<IRunnerServer>();
+                await runnerServer.ConnectAsync(systemConnection.Url, serverCredential);
+                var serverPackages = await runnerServer.GetPackagesAsync("agent", BuildConstants.RunnerPackage.PackageName, 5, includeToken: false, cancellationToken: CancellationToken.None);
+                if (serverPackages.Count > 0)
+                {
+                    serverPackages = serverPackages.OrderByDescending(x => x.Version).ToList();
+                    Trace.Info($"Newer packages {StringUtil.ConvertToJson(serverPackages.Select(x => x.Version.ToString()))}");
+
+                    var warnOnFailedJob = false; // any minor/patch version behind.
+                    var warnOnOldRunnerVersion = false; // >= 2 minor version behind
+                    if (serverPackages.Any(x => x.Version.CompareTo(currentVersion) > 0))
+                    {
+                        Trace.Info($"Current runner version {currentVersion} is behind the latest runner version {serverPackages[0].Version}.");
+                        warnOnFailedJob = true;
+                    }
+
+                    if (serverPackages.Where(x => x.Version.Major == currentVersion.Major && x.Version.Minor > currentVersion.Minor).Count() > 1)
+                    {
+                        Trace.Info($"Current runner version {currentVersion} is way behind the latest runner version {serverPackages[0].Version}.");
+                        warnOnOldRunnerVersion = true;
+                    }
+
+                    if (result == TaskResult.Failed && warnOnFailedJob)
+                    {
+                        jobContext.Warning($"This job failure may be caused by using an out of date version of GitHub runner on your self-hosted runner. You are currently using GitHub runner version {currentVersion}. Please update to the latest version {serverPackages[0].Version}");
+                    }
+                    else if (warnOnOldRunnerVersion)
+                    {
+                        jobContext.Warning($"This self-hosted runner is currently using runner version {currentVersion}. This version is out of date. Please update to the latest version {serverPackages[0].Version}");
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                // Ignore any error since suggest runner update is best effort.
+                Trace.Error($"Caught exception during runner version check: {ex}");
+            }
+        }
     }
 }

src/Runner.Worker/Runner.Worker.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <OutputType>Exe</OutputType>
     <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
     <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
     <Version>$(Version)</Version>
     <PredefinedCulturesOnly>false</PredefinedCulturesOnly>
     <PublishReadyToRunComposite>true</PublishReadyToRunComposite>
@@ -18,9 +19,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
-    <PackageReference Include="System.ServiceProcess.ServiceController" Version="4.4.0" />
-    <PackageReference Include="System.Threading.Channels" Version="4.4.0" />
+    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
+    <PackageReference Include="System.ServiceProcess.ServiceController" Version="8.0.0" />
+    <PackageReference Include="System.Threading.Channels" Version="8.0.0" />
     <PackageReference Include="YamlDotNet.Signed" Version="5.3.0" />
   </ItemGroup>
 

src/Runner.Worker/SnapshotOperationProvider.cs

@@ -0,0 +1,32 @@
+#nullable enable
+using System.IO;
+using System.Threading.Tasks;
+using GitHub.DistributedTask.Pipelines;
+using GitHub.Runner.Common;
+using GitHub.Runner.Sdk;
+namespace GitHub.Runner.Worker;
+
+[ServiceLocator(Default = typeof(SnapshotOperationProvider))]
+public interface ISnapshotOperationProvider : IRunnerService
+{
+    Task CreateSnapshotRequestAsync(IExecutionContext executionContext, Snapshot snapshotRequest);
+}
+
+public class SnapshotOperationProvider : RunnerService, ISnapshotOperationProvider
+{
+    public Task CreateSnapshotRequestAsync(IExecutionContext executionContext, Snapshot snapshotRequest)
+    {
+        var snapshotRequestFilePath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), ".snapshot", "request.json");
+        var snapshotRequestDirectoryPath = Path.GetDirectoryName(snapshotRequestFilePath);
+        if (snapshotRequestDirectoryPath != null)
+        {
+            Directory.CreateDirectory(snapshotRequestDirectoryPath);
+        }
+
+        IOUtil.SaveObject(snapshotRequest, snapshotRequestFilePath);
+        executionContext.Output($"Request written to: {snapshotRequestFilePath}");
+        executionContext.Output("This request will be processed after the job completes. You will not receive any feedback on the snapshot process within the workflow logs of this job.");
+        executionContext.Output("If the snapshot process is successful, you should see a new image with the requested name in the list of available custom images when creating a new GitHub-hosted Runner.");
+        return Task.CompletedTask;
+    }
+}

src/Runner.Worker/StepsRunner.cs

@@ -295,7 +295,7 @@ namespace GitHub.Runner.Worker
                     !jobCancellationToken.IsCancellationRequested)
                 {
                     Trace.Error($"Caught timeout exception from step: {ex.Message}");
-                    step.ExecutionContext.Error("The action has timed out.");
+                    step.ExecutionContext.Error($"The action '{step.DisplayName}' has timed out after {timeoutMinutes} minutes.");
                     step.ExecutionContext.Result = TaskResult.Failed;
                 }
                 else

src/Sdk/Common/Common/Diagnostics/VssHttpEventSource.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Diagnostics.Tracing;
 using System.Globalization;
+using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Net.Sockets;
@@ -335,7 +336,25 @@ namespace GitHub.Services.Common.Diagnostics
             if (IsEnabled())
             {
                 SetActivityId(activity);
-                HttpRequestStop(response.RequestMessage.GetHttpMethod(), response.RequestMessage.RequestUri.AbsoluteUri, (Int32)response.StatusCode);
+                var requestId = "NoExpectedHeader";
+                if (response.Headers != null)
+                {
+                    if (response.Headers.TryGetValues("x-github-request-id", out var headerValues) && headerValues != null)
+                    {
+                        requestId = headerValues.FirstOrDefault();
+                    }
+                    else if (response.Headers.TryGetValues("x-vss-e2eid", out headerValues) && headerValues != null)
+                    {
+                        requestId = headerValues.FirstOrDefault();
+                    }
+
+                    if (string.IsNullOrEmpty(requestId))
+                    {
+                        requestId = "NoExpectedHeader";
+                    }
+                }
+
+                HttpRequestStop(response.RequestMessage.GetHttpMethod(), response.RequestMessage.RequestUri.AbsoluteUri, (Int32)response.StatusCode, requestId);
             }
         }
 
@@ -747,15 +766,16 @@ namespace GitHub.Services.Common.Diagnostics
             }
         }
 
-        [Event(24, Level = EventLevel.Verbose, Task = Tasks.HttpRequest, Opcode = EventOpcode.Stop, Message = "Finished {0} request to {1} with status code {2}")]
+        [Event(24, Level = EventLevel.Verbose, Task = Tasks.HttpRequest, Opcode = EventOpcode.Stop, Message = "Finished {0} request to {1} with status code {2} ({3})")]
         private void HttpRequestStop(
             VssHttpMethod method,
             String url,
-            Int32 statusCode)
+            Int32 statusCode,
+            String requestId)
         {
             if (IsEnabled())
             {
-                WriteEvent(24, (Int32)method, url, statusCode);
+                WriteEvent(24, (Int32)method, url, statusCode, requestId);
             }
         }
 

src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs

@@ -34,6 +34,7 @@ namespace GitHub.Services.Common
 
         public String PropertyName { get; set; }
 
+        [Obsolete]
         [SecurityCritical]
         public override void GetObjectData(SerializationInfo info, StreamingContext context)
         {

src/Sdk/Common/Common/RawHttpHeaders.cs

@@ -1,12 +1,11 @@
 using System;
 using System.ComponentModel;
-using System.Diagnostics.CodeAnalysis;
 
 namespace GitHub.Services.Common.Internal
 {
     [EditorBrowsable(EditorBrowsableState.Never)]
     public static class RawHttpHeaders
     {
-        public const String SessionHeader = "X-Runner-Session";
+        public const String SessionHeader = "X-Actions-Session";
     }
 }