Fix release workflow to use distinct artifact names

* Adding Snapshot additional mapping tokens

* Lint failure fixes

* Lint failure fixes - 2

* Lint failure fixes - 3

* Fixed a few nits

* Lint fixes

* Removed unncessary white space

.devcontainer/devcontainer.json

@@ -4,10 +4,13 @@
   "features": {
     "ghcr.io/devcontainers/features/docker-in-docker:1": {},
     "ghcr.io/devcontainers/features/dotnet": {
-      "version": "6.0.415"
+      "version": "6.0.421"
     },
     "ghcr.io/devcontainers/features/node:1": {
       "version": "16"
+    },
+    "ghcr.io/devcontainers/features/sshd:1": {
+      "version": "latest"
     }
   },
   "customizations": {

.github/workflows/build.yml

@@ -58,29 +58,6 @@ jobs:
         ${{ matrix.devScript }} layout Release ${{ matrix.runtime }}
       working-directory: src
 
-    # Check runtime/externals hash
-    - name: Compute/Compare runtime and externals Hash
-      shell: bash
-      run: |
-        echo "Current dotnet runtime hash result: $DOTNET_RUNTIME_HASH"
-        echo "Current Externals hash result: $EXTERNALS_HASH"
-
-        NeedUpdate=0
-        if [ "$EXTERNALS_HASH" != "$(cat ./src/Misc/contentHash/externals/${{ matrix.runtime }})" ] ;then
-          echo Hash mismatch, Update ./src/Misc/contentHash/externals/${{ matrix.runtime }} to $EXTERNALS_HASH
-          NeedUpdate=1
-        fi
-
-        if [ "$DOTNET_RUNTIME_HASH" != "$(cat ./src/Misc/contentHash/dotnetRuntime/${{ matrix.runtime }})" ] ;then
-          echo Hash mismatch, Update ./src/Misc/contentHash/dotnetRuntime/${{ matrix.runtime }} to $DOTNET_RUNTIME_HASH
-          NeedUpdate=1
-        fi
-
-        exit $NeedUpdate
-      env:
-        DOTNET_RUNTIME_HASH: ${{hashFiles('**/_layout_trims/runtime/**/*')}}
-        EXTERNALS_HASH: ${{hashFiles('**/_layout_trims/externals/**/*')}}
-
     # Run tests
     - name: L0
       run: |
@@ -98,11 +75,8 @@ jobs:
     # Upload runner package tar.gz/zip as artifact
     - name: Publish Artifact
       if: github.event_name != 'pull_request'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: runner-package-${{ matrix.runtime }}
         path: |
           _package
-          _package_trims/trim_externals
-          _package_trims/trim_runtime
-          _package_trims/trim_runtime_externals

.github/workflows/dotnet-upgrade.yml

@@ -84,221 +84,20 @@ jobs:
           git commit -a -m "Upgrade dotnet sdk to v${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}"
           git push --set-upstream origin $branch_name
 
-  build-hashes:
-    if: ${{ needs.dotnet-update.outputs.SHOULD_UPDATE == 1 && needs.dotnet-update.outputs.BRANCH_EXISTS == 0 }}
+  create-pr:
     needs: [dotnet-update]
-    outputs: 
-      # pass outputs from this job to create-pr for use
-      DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION: ${{ needs.dotnet-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}
-      DOTNET_CURRENT_MAJOR_MINOR_VERSION: ${{ needs.dotnet-update.outputs.DOTNET_CURRENT_MAJOR_MINOR_VERSION }}
-      NEEDS_HASH_UPDATE: ${{ steps.compute-hash.outputs.NEED_UPDATE }}
-    strategy:
-      fail-fast: false
-      matrix:
-        runtime: [ linux-x64, linux-arm64, linux-arm, win-x64, win-arm64, osx-x64, osx-arm64 ]
-        include:
-        - runtime: linux-x64
-          os: ubuntu-latest
-          devScript: ./dev.sh
-
-        - runtime: linux-arm64
-          os: ubuntu-latest
-          devScript: ./dev.sh
-
-        - runtime: linux-arm
-          os: ubuntu-latest
-          devScript: ./dev.sh
-
-        - runtime: osx-x64
-          os: macOS-latest
-          devScript: ./dev.sh
-
-        - runtime: osx-arm64
-          os: macOS-latest
-          devScript: ./dev.sh
-
-        - runtime: win-x64
-          os: windows-2019
-          devScript: ./dev
-
-        - runtime: win-arm64
-          os: windows-latest
-          devScript: ./dev
-
-    runs-on: ${{ matrix.os }}
+    if: ${{ needs.dotnet-update.outputs.SHOULD_UPDATE == 1 && needs.dotnet-update.outputs.BRANCH_EXISTS == 0 }}
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3 
+    - uses: actions/checkout@v3
       with:
         ref: feature/dotnetsdk-upgrade/${{ needs.dotnet-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}
-
-    # Build runner layout
-    - name: Build & Layout Release
-      run: |
-        ${{ matrix.devScript }} layout Release ${{ matrix.runtime }}
-      working-directory: src
-
-    # Check runtime/externals hash
-    - name: Compute/Compare runtime and externals Hash
-      id: compute-hash
-      continue-on-error: true
-      shell: bash
-      run: |
-        echo "Current dotnet runtime hash result: $DOTNET_RUNTIME_HASH"
-        echo "Current Externals hash result: $EXTERNALS_HASH"
-
-        NeedUpdate=0
-        if [ "$EXTERNALS_HASH" != "$(cat ./src/Misc/contentHash/externals/${{ matrix.runtime }})" ] ;then
-          echo Hash mismatch, Update ./src/Misc/contentHash/externals/${{ matrix.runtime }} to $EXTERNALS_HASH
-
-          echo "EXTERNAL_HASH=$EXTERNALS_HASH" >> $GITHUB_OUTPUT
-          NeedUpdate=1
-        fi
-
-        if [ "$DOTNET_RUNTIME_HASH" != "$(cat ./src/Misc/contentHash/dotnetRuntime/${{ matrix.runtime }})" ] ;then
-          echo Hash mismatch, Update ./src/Misc/contentHash/dotnetRuntime/${{ matrix.runtime }} to $DOTNET_RUNTIME_HASH
-
-          echo "DOTNET_RUNTIME_HASH=$DOTNET_RUNTIME_HASH" >> $GITHUB_OUTPUT
-          NeedUpdate=1
-        fi
-
-        echo "NEED_UPDATE=$NeedUpdate" >> $GITHUB_OUTPUT
-      env:
-        DOTNET_RUNTIME_HASH: ${{hashFiles('**/_layout_trims/runtime/**/*')}}
-        EXTERNALS_HASH: ${{hashFiles('**/_layout_trims/externals/**/*')}}
-    - name: update hash
-      if: ${{ steps.compute-hash.outputs.NEED_UPDATE == 1 }}
-      shell: bash
-      run: |
-        ExternalHash=${{ steps.compute-hash.outputs.EXTERNAL_HASH }}
-        DotNetRuntimeHash=${{ steps.compute-hash.outputs.DOTNET_RUNTIME_HASH }}
-
-        if [ -n "$ExternalHash" ]; then
-          echo "$ExternalHash" > ./src/Misc/contentHash/externals/${{ matrix.runtime }}
-        fi
-
-        if [ -n "$DotNetRuntimeHash" ]; then
-          echo "$DotNetRuntimeHash" > ./src/Misc/contentHash/dotnetRuntime/${{ matrix.runtime }}
-        fi
-    - name: cache updated hashes
-      if: ${{ steps.compute-hash.outputs.NEED_UPDATE == 1 }}
-      uses: actions/cache/save@v3
-      with:
-        enableCrossOsArchive: true
-        path: |
-          ./src/Misc/contentHash/externals/${{ matrix.runtime }}
-          ./src/Misc/contentHash/dotnetRuntime/${{ matrix.runtime }}
-        key: compute-hashes-${{ matrix.runtime }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
-
-
-  hash-update:
-    needs: [build-hashes]
-    if: ${{ needs.build-hashes.outputs.NEEDS_HASH_UPDATE == 1 }}
-    outputs: 
-      # pass outputs from this job to create-pr for use
-      DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION: ${{ needs.build-hashes.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}
-      DOTNET_CURRENT_MAJOR_MINOR_VERSION: ${{ needs.build-hashes.outputs.DOTNET_CURRENT_MAJOR_MINOR_VERSION }}
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-      with: 
-        ref: feature/dotnetsdk-upgrade/${{ needs.build-hashes.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}
-    - name: Restore cached hashes - linux-x64
-      id: cache-restore-linux-x64
-      uses: actions/cache/restore@v3
-      with:
-        enableCrossOsArchive: true
-        path: |
-          ./src/Misc/contentHash/externals/linux-x64
-          ./src/Misc/contentHash/dotnetRuntime/linux-x64
-        key: compute-hashes-linux-x64-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
-    - name: Restore cached hashes - linux-arm64
-      id: cache-restore-linux-arm64
-      uses: actions/cache/restore@v3
-      with:
-        enableCrossOsArchive: true
-        path: |
-          ./src/Misc/contentHash/externals/linux-arm64
-          ./src/Misc/contentHash/dotnetRuntime/linux-arm64
-        key: compute-hashes-linux-arm64-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
-    - name: Restore cached hashes - linux-arm
-      id: cache-restore-linux-arm
-      uses: actions/cache/restore@v3
-      with:
-        enableCrossOsArchive: true
-        path: |
-          ./src/Misc/contentHash/externals/linux-arm
-          ./src/Misc/contentHash/dotnetRuntime/linux-arm
-        key: compute-hashes-linux-arm-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
-    - name: Restore cached hashes - osx-x64
-      id: cache-restore-osx-x64
-      uses: actions/cache/restore@v3
-      with:
-        enableCrossOsArchive: true
-        path: |
-          ./src/Misc/contentHash/externals/osx-x64
-          ./src/Misc/contentHash/dotnetRuntime/osx-x64
-        key: compute-hashes-osx-x64-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
-    - name: Restore cached hashes - osx-arm64
-      id: cache-restore-osx-arm64
-      uses: actions/cache/restore@v3
-      with:
-        enableCrossOsArchive: true
-        path: |
-          ./src/Misc/contentHash/externals/osx-arm64
-          ./src/Misc/contentHash/dotnetRuntime/osx-arm64
-        key: compute-hashes-osx-arm64-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
-    - name: Restore cached hashes - win-x64
-      id: cache-restore-win-x64
-      uses: actions/cache/restore@v3
-      with:
-        enableCrossOsArchive: true
-        path: |
-          ./src/Misc/contentHash/externals/win-x64
-          ./src/Misc/contentHash/dotnetRuntime/win-x64
-        key: compute-hashes-win-x64-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
-    - name: Restore cached hashes - win-arm64
-      id: cache-restore-win-arm64
-      uses: actions/cache/restore@v3
-      with:
-        enableCrossOsArchive: true
-        path: |
-          ./src/Misc/contentHash/externals/win-arm64
-          ./src/Misc/contentHash/dotnetRuntime/win-arm64
-        key: compute-hashes-win-arm64-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
-    - name: Fetch cached computed hashes
-      if: steps.cache-restore-linux-x64.outputs.cache-hit == 'true' ||
-          steps.cache-restore-linux-arm64.outputs.cache-hit == 'true' ||
-          steps.cache-restore-linux-arm.outputs.cache-hit == 'true' ||
-          steps.cache-restore-win-x64.outputs.cache-hit == 'true' ||
-          steps.cache-restore-win-arm64.outputs.cache-hit == 'true' ||
-          steps.cache-restore-osx-x64.outputs.cache-hit == 'true' ||
-          steps.cache-restore-osx-arm64.outputs.cache-hit == 'true'
-      shell: bash
-      run: |
-        Environments=( "linux-x64" "linux-arm64" "linux-arm" "win-x64" "win-arm64" "osx-x64" "osx-arm64" )
-
-        git config --global user.name "github-actions[bot]"
-        git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
-        git commit -a -m "Update computed hashes"
-        git push --set-upstream origin feature/dotnetsdk-upgrade/${{ needs.build-hashes.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}
-
-  create-pr:
-    needs: [hash-update]
-    outputs:
-      # pass outputs from this job to run-tests for use
-      DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION: ${{ needs.hash-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}
-      DOTNET_CURRENT_MAJOR_MINOR_VERSION: ${{ needs.hash-update.outputs.DOTNET_CURRENT_MAJOR_MINOR_VERSION }}
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: feature/dotnetsdk-upgrade/${{ needs.hash-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}
     - name: Create Pull Request
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
-        gh pr create -B main -H feature/dotnetsdk-upgrade/${{ needs.hash-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }} --title "Update dotnet sdk to latest version @${{ needs.hash-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}" --body "
-          https://dotnetcli.blob.core.windows.net/dotnet/Sdk/${{ needs.hash-update.outputs.DOTNET_CURRENT_MAJOR_MINOR_VERSION }}/latest.version
+        gh pr create -B main -H feature/dotnetsdk-upgrade/${{ needs.dotnet-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }} --title "Update dotnet sdk to latest version @${{ needs.dotnet-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}" --body "
+          https://dotnetcli.blob.core.windows.net/dotnet/Sdk/${{ needs.dotnet-update.outputs.DOTNET_CURRENT_MAJOR_MINOR_VERSION }}/latest.version
           
             
           ---

.github/workflows/publish-image.yml

@@ -25,10 +25,12 @@ jobs:
       - name: Compute image version
         id: image
         uses: actions/github-script@v6
+        env:
+          RUNNER_VERSION: ${{ github.event.inputs.runnerVersion }}
         with:
           script: |
             const fs = require('fs');
-            const inputRunnerVersion = "${{ github.event.inputs.runnerVersion }}"
+            const inputRunnerVersion = process.env.RUNNER_VERSION;
             if (inputRunnerVersion) {
               console.log(`Using input runner version ${inputRunnerVersion}`)
               core.setOutput('version', inputRunnerVersion);

.github/workflows/release.yml

@@ -53,27 +53,6 @@ jobs:
       win-arm64-sha: ${{ steps.sha.outputs.win-arm64-sha256 }}
       osx-x64-sha: ${{ steps.sha.outputs.osx-x64-sha256 }}
       osx-arm64-sha: ${{ steps.sha.outputs.osx-arm64-sha256 }}
-      linux-x64-sha-noexternals: ${{ steps.sha_noexternals.outputs.linux-x64-sha256 }}
-      linux-arm64-sha-noexternals: ${{ steps.sha_noexternals.outputs.linux-arm64-sha256 }}
-      linux-arm-sha-noexternals: ${{ steps.sha_noexternals.outputs.linux-arm-sha256 }}
-      win-x64-sha-noexternals: ${{ steps.sha_noexternals.outputs.win-x64-sha256 }}
-      win-arm64-sha-noexternals: ${{ steps.sha_noexternals.outputs.win-arm64-sha256 }}
-      osx-x64-sha-noexternals: ${{ steps.sha_noexternals.outputs.osx-x64-sha256 }}
-      osx-arm64-sha-noexternals: ${{ steps.sha_noexternals.outputs.osx-arm64-sha256 }}
-      linux-x64-sha-noruntime: ${{ steps.sha_noruntime.outputs.linux-x64-sha256 }}
-      linux-arm64-sha-noruntime: ${{ steps.sha_noruntime.outputs.linux-arm64-sha256 }}
-      linux-arm-sha-noruntime: ${{ steps.sha_noruntime.outputs.linux-arm-sha256 }}
-      win-x64-sha-noruntime: ${{ steps.sha_noruntime.outputs.win-x64-sha256 }}
-      win-arm64-sha-noruntime: ${{ steps.sha_noruntime.outputs.win-arm64-sha256 }}
-      osx-x64-sha-noruntime: ${{ steps.sha_noruntime.outputs.osx-x64-sha256 }}
-      osx-arm64-sha-noruntime: ${{ steps.sha_noruntime.outputs.osx-arm64-sha256 }}
-      linux-x64-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.linux-x64-sha256 }}
-      linux-arm64-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.linux-arm64-sha256 }}
-      linux-arm-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.linux-arm-sha256 }}
-      win-x64-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.win-x64-sha256 }}
-      win-arm64-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.win-arm64-sha256 }}
-      osx-x64-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.osx-x64-sha256 }}
-      osx-arm64-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.osx-arm64-sha256 }}
     strategy:
       matrix:
         runtime: [ linux-x64, linux-arm64, linux-arm, win-x64, osx-x64, osx-arm64, win-arm64 ]
@@ -136,90 +115,15 @@ jobs:
       id: sha
       name: Compute SHA256
       working-directory: _package
-    - run: |
-        file=$(ls)
-        sha=$(sha256sum $file | awk '{ print $1 }')
-        echo "Computed sha256: $sha for $file"
-        echo "${{matrix.runtime}}-sha256=$sha" >> $GITHUB_OUTPUT
-        echo "sha256=$sha" >> $GITHUB_OUTPUT
-      shell: bash
-      id: sha_noexternals
-      name: Compute SHA256
-      working-directory: _package_trims/trim_externals
-    - run: |
-        file=$(ls)
-        sha=$(sha256sum $file | awk '{ print $1 }')
-        echo "Computed sha256: $sha for $file"
-        echo "${{matrix.runtime}}-sha256=$sha" >> $GITHUB_OUTPUT
-        echo "sha256=$sha" >> $GITHUB_OUTPUT
-      shell: bash
-      id: sha_noruntime
-      name: Compute SHA256
-      working-directory: _package_trims/trim_runtime
-    - run: |
-        file=$(ls)
-        sha=$(sha256sum $file | awk '{ print $1 }')
-        echo "Computed sha256: $sha for $file"
-        echo "${{matrix.runtime}}-sha256=$sha" >> $GITHUB_OUTPUT
-        echo "sha256=$sha" >> $GITHUB_OUTPUT
-      shell: bash
-      id: sha_noruntime_noexternals
-      name: Compute SHA256
-      working-directory: _package_trims/trim_runtime_externals
-
-    - name: Create trimmedpackages.json for ${{ matrix.runtime }}
-      if: matrix.runtime == 'win-x64' || matrix.runtime == 'win-arm64'
-      uses: actions/github-script@0.3.0
-      with:
-        github-token: ${{secrets.GITHUB_TOKEN}}
-        script: |
-          const core = require('@actions/core')
-          const fs = require('fs');
-          const runnerVersion = fs.readFileSync('src/runnerversion', 'utf8').replace(/\n$/g, '')
-          var trimmedPackages = fs.readFileSync('src/Misc/trimmedpackages_zip.json', 'utf8').replace(/<RUNNER_VERSION>/g, runnerVersion).replace(/<RUNNER_PLATFORM>/g, '${{ matrix.runtime }}')
-          trimmedPackages = trimmedPackages.replace(/<RUNTIME_HASH>/g, '${{hashFiles('**/_layout_trims/runtime/**/*')}}')
-          trimmedPackages = trimmedPackages.replace(/<EXTERNALS_HASH>/g, '${{hashFiles('**/_layout_trims/externals/**/*')}}')
-
-          trimmedPackages = trimmedPackages.replace(/<NO_RUNTIME_EXTERNALS_HASH>/g, '${{steps.sha_noruntime_noexternals.outputs.sha256}}')
-          trimmedPackages = trimmedPackages.replace(/<NO_RUNTIME_HASH>/g, '${{steps.sha_noruntime.outputs.sha256}}')
-          trimmedPackages = trimmedPackages.replace(/<NO_EXTERNALS_HASH>/g, '${{steps.sha_noexternals.outputs.sha256}}')
-
-          console.log(trimmedPackages)
-          fs.writeFileSync('${{ matrix.runtime }}-trimmedpackages.json', trimmedPackages)
-
-    - name: Create trimmedpackages.json for ${{ matrix.runtime }}
-      if: matrix.runtime != 'win-x64' && matrix.runtime != 'win-arm64'
-      uses: actions/github-script@0.3.0
-      with:
-        github-token: ${{secrets.GITHUB_TOKEN}}
-        script: |
-          const core = require('@actions/core')
-          const fs = require('fs');
-          const runnerVersion = fs.readFileSync('src/runnerversion', 'utf8').replace(/\n$/g, '')
-          var trimmedPackages = fs.readFileSync('src/Misc/trimmedpackages_targz.json', 'utf8').replace(/<RUNNER_VERSION>/g, runnerVersion).replace(/<RUNNER_PLATFORM>/g, '${{ matrix.runtime }}')
-          trimmedPackages = trimmedPackages.replace(/<RUNTIME_HASH>/g, '${{hashFiles('**/_layout_trims/runtime/**/*')}}')
-          trimmedPackages = trimmedPackages.replace(/<EXTERNALS_HASH>/g, '${{hashFiles('**/_layout_trims/externals/**/*')}}')
-
-          trimmedPackages = trimmedPackages.replace(/<NO_RUNTIME_EXTERNALS_HASH>/g, '${{steps.sha_noruntime_noexternals.outputs.sha256}}')
-          trimmedPackages = trimmedPackages.replace(/<NO_RUNTIME_HASH>/g, '${{steps.sha_noruntime.outputs.sha256}}')
-          trimmedPackages = trimmedPackages.replace(/<NO_EXTERNALS_HASH>/g, '${{steps.sha_noexternals.outputs.sha256}}')
-
-          console.log(trimmedPackages)
-          fs.writeFileSync('${{ matrix.runtime }}-trimmedpackages.json', trimmedPackages)
 
     # Upload runner package tar.gz/zip as artifact.
-    # Since each package name is unique, so we don't need to put ${{matrix}} info into artifact name
     - name: Publish Artifact
       if: github.event_name != 'pull_request'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
-        name: runner-packages
+        name: runner-packages-${{ matrix.runtime }}
         path: |
           _package
-          _package_trims/trim_externals
-          _package_trims/trim_runtime
-          _package_trims/trim_runtime_externals
-          ${{ matrix.runtime }}-trimmedpackages.json
 
   release:
     needs: build
@@ -229,10 +133,40 @@ jobs:
     - uses: actions/checkout@v3
 
     # Download runner package tar.gz/zip produced by 'build' job
-    - name: Download Artifact
-      uses: actions/download-artifact@v1
+    - name: Download Artifact (win-x64)
+      uses: actions/download-artifact@v4
       with:
-        name: runner-packages
+        name: runner-packages-win-x64
+        path: ./
+    - name: Download Artifact (win-arm64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-win-arm64
+        path: ./
+    - name: Download Artifact (osx-x64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-osx-x64
+        path: ./
+    - name: Download Artifact (osx-arm64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-osx-arm64
+        path: ./
+    - name: Download Artifact (linux-x64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-linux-x64
+        path: ./
+    - name: Download Artifact (linux-arm)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-linux-arm
+        path: ./
+    - name: Download Artifact (linux-arm64)
+      uses: actions/download-artifact@v4
+      with:
+        name: runner-packages-linux-arm64
         path: ./
 
     # Create ReleaseNote file
@@ -253,33 +187,11 @@ jobs:
           releaseNote = releaseNote.replace(/<LINUX_X64_SHA>/g, '${{needs.build.outputs.linux-x64-sha}}')
           releaseNote = releaseNote.replace(/<LINUX_ARM_SHA>/g, '${{needs.build.outputs.linux-arm-sha}}')
           releaseNote = releaseNote.replace(/<LINUX_ARM64_SHA>/g, '${{needs.build.outputs.linux-arm64-sha}}')
-          releaseNote = releaseNote.replace(/<WIN_X64_SHA_NOEXTERNALS>/g, '${{needs.build.outputs.win-x64-sha-noexternals}}')
-          releaseNote = releaseNote.replace(/<WIN_ARM64_SHA_NOEXTERNALS>/g, '${{needs.build.outputs.win-arm64-sha-noexternals}}')
-          releaseNote = releaseNote.replace(/<OSX_X64_SHA_NOEXTERNALS>/g, '${{needs.build.outputs.osx-x64-sha-noexternals}}')
-          releaseNote = releaseNote.replace(/<OSX_ARM64_SHA_NOEXTERNALS>/g, '${{needs.build.outputs.osx-arm64-sha-noexternals}}')
-          releaseNote = releaseNote.replace(/<LINUX_X64_SHA_NOEXTERNALS>/g, '${{needs.build.outputs.linux-x64-sha-noexternals}}')
-          releaseNote = releaseNote.replace(/<LINUX_ARM_SHA_NOEXTERNALS>/g, '${{needs.build.outputs.linux-arm-sha-noexternals}}')
-          releaseNote = releaseNote.replace(/<LINUX_ARM64_SHA_NOEXTERNALS>/g, '${{needs.build.outputs.linux-arm64-sha-noexternals}}')
-          releaseNote = releaseNote.replace(/<WIN_X64_SHA_NORUNTIME>/g, '${{needs.build.outputs.win-x64-sha-noruntime}}')
-          releaseNote = releaseNote.replace(/<WIN_ARM64_SHA_NORUNTIME>/g, '${{needs.build.outputs.win-arm64-sha-noruntime}}')
-          releaseNote = releaseNote.replace(/<OSX_X64_SHA_NORUNTIME>/g, '${{needs.build.outputs.osx-x64-sha-noruntime}}')
-          releaseNote = releaseNote.replace(/<OSX_ARM64_SHA_NORUNTIME>/g, '${{needs.build.outputs.osx-arm64-sha-noruntime}}')
-          releaseNote = releaseNote.replace(/<LINUX_X64_SHA_NORUNTIME>/g, '${{needs.build.outputs.linux-x64-sha-noruntime}}')
-          releaseNote = releaseNote.replace(/<LINUX_ARM_SHA_NORUNTIME>/g, '${{needs.build.outputs.linux-arm-sha-noruntime}}')
-          releaseNote = releaseNote.replace(/<LINUX_ARM64_SHA_NORUNTIME>/g, '${{needs.build.outputs.linux-arm64-sha-noruntime}}')
-          releaseNote = releaseNote.replace(/<WIN_X64_SHA_NORUNTIME_NOEXTERNALS>/g, '${{needs.build.outputs.win-x64-sha-noruntime-noexternals}}')
-          releaseNote = releaseNote.replace(/<WIN_ARM64_SHA_NORUNTIME_NOEXTERNALS>/g, '${{needs.build.outputs.win-arm64-sha-noruntime-noexternals}}')
-          releaseNote = releaseNote.replace(/<OSX_X64_SHA_NORUNTIME_NOEXTERNALS>/g, '${{needs.build.outputs.osx-x64-sha-noruntime-noexternals}}')
-          releaseNote = releaseNote.replace(/<OSX_ARM64_SHA_NORUNTIME_NOEXTERNALS>/g, '${{needs.build.outputs.osx-arm64-sha-noruntime-noexternals}}')
-          releaseNote = releaseNote.replace(/<LINUX_X64_SHA_NORUNTIME_NOEXTERNALS>/g, '${{needs.build.outputs.linux-x64-sha-noruntime-noexternals}}')
-          releaseNote = releaseNote.replace(/<LINUX_ARM_SHA_NORUNTIME_NOEXTERNALS>/g, '${{needs.build.outputs.linux-arm-sha-noruntime-noexternals}}')
-          releaseNote = releaseNote.replace(/<LINUX_ARM64_SHA_NORUNTIME_NOEXTERNALS>/g, '${{needs.build.outputs.linux-arm64-sha-noruntime-noexternals}}')
           console.log(releaseNote)
           core.setOutput('version', runnerVersion);
           core.setOutput('note', releaseNote);
 
     - name: Validate Packages HASH
-      working-directory: _package
       run: |
         ls -l
         echo "${{needs.build.outputs.win-x64-sha}}  actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip" | shasum -a 256 -c
@@ -309,7 +221,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip
+        asset_path: ${{ github.workspace }}/actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip
         asset_name: actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip
         asset_content_type: application/octet-stream
 
@@ -319,7 +231,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}.zip
+        asset_path: ${{ github.workspace }}/actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}.zip
         asset_name: actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}.zip
         asset_content_type: application/octet-stream
 
@@ -329,7 +241,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
@@ -339,7 +251,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
@@ -349,7 +261,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
@@ -359,7 +271,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
@@ -369,298 +281,10 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
-    # Upload release assets (trim externals)
-    - name: Upload Release Asset (win-x64-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_externals/actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}-noexternals.zip
-        asset_name: actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}-noexternals.zip
-        asset_content_type: application/octet-stream
-
-    # Upload release assets (trim externals)
-    - name: Upload Release Asset (win-arm64-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_externals/actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-noexternals.zip
-        asset_name: actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-noexternals.zip
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (linux-x64-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_externals/actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}-noexternals.tar.gz
-        asset_name: actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}-noexternals.tar.gz
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (osx-x64-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_externals/actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}-noexternals.tar.gz
-        asset_name: actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}-noexternals.tar.gz
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (osx-arm64-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_externals/actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}-noexternals.tar.gz
-        asset_name: actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}-noexternals.tar.gz
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (linux-arm-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_externals/actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}-noexternals.tar.gz
-        asset_name: actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}-noexternals.tar.gz
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (linux-arm64-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_externals/actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}-noexternals.tar.gz
-        asset_name: actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}-noexternals.tar.gz
-        asset_content_type: application/octet-stream
-
-    # Upload release assets (trim runtime)
-    - name: Upload Release Asset (win-x64-noruntime)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime/actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}-noruntime.zip
-        asset_name: actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}-noruntime.zip
-        asset_content_type: application/octet-stream
-
-    # Upload release assets (trim runtime)
-    - name: Upload Release Asset (win-arm64-noruntime)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime/actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-noruntime.zip
-        asset_name: actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-noruntime.zip
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (linux-x64-noruntime)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime/actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}-noruntime.tar.gz
-        asset_name: actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}-noruntime.tar.gz
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (osx-x64-noruntime)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime/actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}-noruntime.tar.gz
-        asset_name: actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}-noruntime.tar.gz
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (osx-arm64-noruntime)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime/actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}-noruntime.tar.gz
-        asset_name: actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}-noruntime.tar.gz
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (linux-arm-noruntime)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime/actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}-noruntime.tar.gz
-        asset_name: actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}-noruntime.tar.gz
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (linux-arm64-noruntime)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime/actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}-noruntime.tar.gz
-        asset_name: actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}-noruntime.tar.gz
-        asset_content_type: application/octet-stream
-
-    # Upload release assets (trim runtime and externals)
-    - name: Upload Release Asset (win-x64-noruntime-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime_externals/actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.zip
-        asset_name: actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.zip
-        asset_content_type: application/octet-stream
-
-    # Upload release assets (trim runtime and externals)
-    - name: Upload Release Asset (win-arm64-noruntime-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime_externals/actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.zip
-        asset_name: actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.zip
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (linux-x64-noruntime-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime_externals/actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.tar.gz
-        asset_name: actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.tar.gz
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (osx-x64-noruntime-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime_externals/actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.tar.gz
-        asset_name: actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.tar.gz
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (osx-arm64-noruntime-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime_externals/actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.tar.gz
-        asset_name: actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.tar.gz
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (linux-arm-noruntime-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime_externals/actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.tar.gz
-        asset_name: actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.tar.gz
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (linux-arm64-noruntime-noexternals)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package_trims/trim_runtime_externals/actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.tar.gz
-        asset_name: actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.tar.gz
-        asset_content_type: application/octet-stream
-
-    # Upload release assets (trimmedpackages.json)
-    - name: Upload Release Asset (win-x64-trimmedpackages.json)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/win-x64-trimmedpackages.json
-        asset_name: actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}-trimmedpackages.json
-        asset_content_type: application/octet-stream
-
-    # Upload release assets (trimmedpackages.json)
-    - name: Upload Release Asset (win-arm64-trimmedpackages.json)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/win-arm64-trimmedpackages.json
-        asset_name: actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-trimmedpackages.json
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (linux-x64-trimmedpackages.json)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/linux-x64-trimmedpackages.json
-        asset_name: actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}-trimmedpackages.json
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (osx-x64-trimmedpackages.json)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/osx-x64-trimmedpackages.json
-        asset_name: actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}-trimmedpackages.json
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (osx-arm64-trimmedpackages.json)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/osx-arm64-trimmedpackages.json
-        asset_name: actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}-trimmedpackages.json
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (linux-arm-trimmedpackages.json)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/linux-arm-trimmedpackages.json
-        asset_name: actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}-trimmedpackages.json
-        asset_content_type: application/octet-stream
-
-    - name: Upload Release Asset (linux-arm64-trimmedpackages.json)
-      uses: actions/upload-release-asset@v1.0.1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/linux-arm64-trimmedpackages.json
-        asset_name: actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}-trimmedpackages.json
-        asset_content_type: application/octet-stream
-
   publish-image:
     needs: release
     runs-on: ubuntu-latest

docs/start/envlinux.md

@@ -4,16 +4,7 @@
 
 ## Supported Distributions and Versions
 
-x64
-  - Red Hat Enterprise Linux 7+
-  - CentOS 7+
-  - Oracle Linux 7+
-  - Fedora 29+
-  - Debian 9+
-  - Ubuntu 16.04+
-  - Linux Mint 18+
-  - openSUSE 15+
-  - SUSE Enterprise Linux (SLES) 12 SP2+
+Please see "[Supported architectures and operating systems for self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#linux)."
 
 ## Install .Net Core 3.x Linux Dependencies
 

docs/start/envosx.md

@@ -4,7 +4,6 @@
 
 ## Supported Versions
 
-  - macOS High Sierra (10.13) and later versions
-  - x64 and arm64 (Apple Silicon)
+Please see "[Supported architectures and operating systems for self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#macos)."
 
 ## [More .Net Core Prerequisites Information](https://docs.microsoft.com/en-us/dotnet/core/macos-prerequisites?tabs=netcore30)

docs/start/envwin.md

@@ -2,11 +2,6 @@
 
 ## Supported Versions
 
- - Windows 7 64-bit
- - Windows 8.1 64-bit
- - Windows 10 64-bit
- - Windows Server 2012 R2 64-bit
- - Windows Server 2016 64-bit
- - Windows Server 2019 64-bit
+Please see "[Supported architectures and operating systems for self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#windows)."
 
 ## [More .NET Core Prerequisites Information](https://docs.microsoft.com/en-us/dotnet/core/windows-prerequisites?tabs=netcore30)

images/Dockerfile

@@ -4,9 +4,9 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0-jammy as build
 ARG TARGETOS
 ARG TARGETARCH
 ARG RUNNER_VERSION
-ARG RUNNER_CONTAINER_HOOKS_VERSION=0.5.0
-ARG DOCKER_VERSION=24.0.6
-ARG BUILDX_VERSION=0.11.2
+ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.1
+ARG DOCKER_VERSION=27.1.1
+ARG BUILDX_VERSION=0.16.2
 
 RUN apt update -y && apt install curl unzip -y
 
@@ -39,12 +39,16 @@ ENV RUNNER_MANUALLY_TRAP_SIG=1
 ENV ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT=1
 ENV ImageOS=ubuntu22
 
-RUN apt-get update -y \
-    && apt-get install -y --no-install-recommends \
-    sudo \
-    lsb-release \
+# 'gpg-agent' and 'software-properties-common' are needed for the 'add-apt-repository' command that follows
+RUN apt update -y \
+    && apt install -y --no-install-recommends sudo lsb-release gpg-agent software-properties-common curl jq unzip \
     && rm -rf /var/lib/apt/lists/*
 
+# Configure git-core/ppa based on guidance here:  https://git-scm.com/download/linux
+RUN add-apt-repository ppa:git-core/ppa \
+    && apt update -y \
+    && apt install -y --no-install-recommends git
+
 RUN adduser --disabled-password --gecos "" --uid 1001 runner \
     && groupadd docker --gid 123 \
     && usermod -aG sudo runner \

releaseNote.md

@@ -1,30 +1,29 @@
 ## What's Changed
-* Trim whitespace in `./Misc/contentHash/dotnetRuntime/*` by @TingluoHuang in https://github.com/actions/runner/pull/2915
-* Send os and arch during long poll by @luketomlinson in https://github.com/actions/runner/pull/2913
-* Revert "Update default version to node20 (#2844)" by @takost in https://github.com/actions/runner/pull/2918
-* Fix telemetry publish from JobServerQueue. by @TingluoHuang in https://github.com/actions/runner/pull/2919
-* Use block blob instead of append blob  by @yacaovsnc in https://github.com/actions/runner/pull/2924
-* Provide detail info on untar failures. by @TingluoHuang in https://github.com/actions/runner/pull/2939
-* Bump node.js to 20.8.1 by @TingluoHuang in https://github.com/actions/runner/pull/2945
-* Update dotnet sdk to latest version @6.0.415 by @github-actions in https://github.com/actions/runner/pull/2929
-* Fix typo in log strings by @rajbos in https://github.com/actions/runner/pull/2695
-* feat: add support of arm64 arch runners in service creation script by @tuxity in https://github.com/actions/runner/pull/2606
-* Add `buildx` to images by @ajschmidt8 in https://github.com/actions/runner/pull/2901
 
-## New Contributors
-* @tuxity made their first contribution in https://github.com/actions/runner/pull/2606
+- Adding Snapshot additional mapping tokens https://github.com/actions/runner/pull/3468
+- Create launch httpclient using the right handler and setting https://github.com/actions/runner/pull/3476
+- Fix missing default user-agent for jitconfig runner https://github.com/actions/runner/pull/3473
+- Cleanup back-compat code for interpreting Run Service status codes https://github.com/actions/runner/pull/3456
+- Add runner or worker to the useragent https://github.com/actions/runner/pull/3457
+- Handle Error Body in Responses from Broker https://github.com/actions/runner/pull/3454
+- Fix issues for composite actions (Run Service flow) https://github.com/actions/runner/pull/3446
+- Trace GitHub RequestId to log https://github.com/actions/runner/pull/3442
+- Add `jq`, `git`, `unzip` and `curl` to default packages installed https://github.com/actions/runner/pull/3056
+- Add pid to user-agent and session owner https://github.com/actions/runner/pull/3432
 
-**Full Changelog**: https://github.com/actions/runner/compare/v2.310.2...v2.311.0
+**Full Changelog**: https://github.com/actions/runner/compare/v2.319.1...v2.320.0
 
-_Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. 
-To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. 
+_Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet.
+To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository.
 See https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/adding-self-hosted-runners_
 
 ## Windows x64
+
 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.
 
 The following snipped needs to be run on `powershell`:
-``` powershell
+
+```powershell
 # Create a folder under the drive root
 mkdir \actions-runner ; cd \actions-runner
 # Download the latest runner package
@@ -35,12 +34,14 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem ;
 ```
 
 ## [Pre-release] Windows arm64
+
 **Warning:** Windows arm64 runners are currently in preview status and use [unofficial versions of nodejs](https://unofficial-builds.nodejs.org/). They are not intended for production workflows.
 
 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.
 
 The following snipped needs to be run on `powershell`:
-``` powershell
+
+```powershell
 # Create a folder under the drive root
 mkdir \actions-runner ; cd \actions-runner
 # Download the latest runner package
@@ -52,7 +53,7 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem ;
 
 ## OSX x64
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -63,7 +64,7 @@ tar xzf ./actions-runner-osx-x64-<RUNNER_VERSION>.tar.gz
 
 ## OSX arm64 (Apple silicon)
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -74,7 +75,7 @@ tar xzf ./actions-runner-osx-arm64-<RUNNER_VERSION>.tar.gz
 
 ## Linux x64
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -85,7 +86,7 @@ tar xzf ./actions-runner-linux-x64-<RUNNER_VERSION>.tar.gz
 
 ## Linux arm64
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -96,7 +97,7 @@ tar xzf ./actions-runner-linux-arm64-<RUNNER_VERSION>.tar.gz
 
 ## Linux arm
 
-``` bash
+```bash
 # Create a folder
 mkdir actions-runner && cd actions-runner
 # Download the latest runner package
@@ -106,6 +107,7 @@ tar xzf ./actions-runner-linux-arm-<RUNNER_VERSION>.tar.gz
 ```
 
 ## Using your self hosted runner
+
 For additional details about configuring, running, or shutting down the runner please check out our [product docs.](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/adding-self-hosted-runners)
 
 ## SHA-256 Checksums
@@ -119,27 +121,3 @@ The SHA-256 checksums for the packages included in this build are shown below:
 - actions-runner-linux-x64-<RUNNER_VERSION>.tar.gz <!-- BEGIN SHA linux-x64 --><LINUX_X64_SHA><!-- END SHA linux-x64 -->
 - actions-runner-linux-arm64-<RUNNER_VERSION>.tar.gz <!-- BEGIN SHA linux-arm64 --><LINUX_ARM64_SHA><!-- END SHA linux-arm64 -->
 - actions-runner-linux-arm-<RUNNER_VERSION>.tar.gz <!-- BEGIN SHA linux-arm --><LINUX_ARM_SHA><!-- END SHA linux-arm -->
-
-- actions-runner-win-x64-<RUNNER_VERSION>-noexternals.zip <!-- BEGIN SHA win-x64_noexternals --><WIN_X64_SHA_NOEXTERNALS><!-- END SHA win-x64_noexternals -->
-- actions-runner-win-arm64-<RUNNER_VERSION>-noexternals.zip <!-- BEGIN SHA win-arm64_noexternals --><WIN_ARM64_SHA_NOEXTERNALS><!-- END SHA win-arm64_noexternals -->
-- actions-runner-osx-x64-<RUNNER_VERSION>-noexternals.tar.gz <!-- BEGIN SHA osx-x64_noexternals --><OSX_X64_SHA_NOEXTERNALS><!-- END SHA osx-x64_noexternals -->
-- actions-runner-osx-arm64-<RUNNER_VERSION>-noexternals.tar.gz <!-- BEGIN SHA osx-arm64_noexternals --><OSX_ARM64_SHA_NOEXTERNALS><!-- END SHA osx-arm64_noexternals -->
-- actions-runner-linux-x64-<RUNNER_VERSION>-noexternals.tar.gz <!-- BEGIN SHA linux-x64_noexternals --><LINUX_X64_SHA_NOEXTERNALS><!-- END SHA linux-x64_noexternals -->
-- actions-runner-linux-arm64-<RUNNER_VERSION>-noexternals.tar.gz <!-- BEGIN SHA linux-arm64_noexternals --><LINUX_ARM64_SHA_NOEXTERNALS><!-- END SHA linux-arm64_noexternals -->
-- actions-runner-linux-arm-<RUNNER_VERSION>-noexternals.tar.gz <!-- BEGIN SHA linux-arm_noexternals --><LINUX_ARM_SHA_NOEXTERNALS><!-- END SHA linux-arm_noexternals -->
-
-- actions-runner-win-x64-<RUNNER_VERSION>-noruntime.zip <!-- BEGIN SHA win-x64_noruntime --><WIN_X64_SHA_NORUNTIME><!-- END SHA win-x64_noruntime -->
-- actions-runner-win-arm64-<RUNNER_VERSION>-noruntime.zip <!-- BEGIN SHA win-arm64_noruntime --><WIN_ARM64_SHA_NORUNTIME><!-- END SHA win-arm64_noruntime -->
-- actions-runner-osx-x64-<RUNNER_VERSION>-noruntime.tar.gz <!-- BEGIN SHA osx-x64_noruntime --><OSX_X64_SHA_NORUNTIME><!-- END SHA osx-x64_noruntime -->
-- actions-runner-osx-arm64-<RUNNER_VERSION>-noruntime.tar.gz <!-- BEGIN SHA osx-arm64_noruntime --><OSX_ARM64_SHA_NORUNTIME><!-- END SHA osx-arm64_noruntime -->
-- actions-runner-linux-x64-<RUNNER_VERSION>-noruntime.tar.gz <!-- BEGIN SHA linux-x64_noruntime --><LINUX_X64_SHA_NORUNTIME><!-- END SHA linux-x64_noruntime -->
-- actions-runner-linux-arm64-<RUNNER_VERSION>-noruntime.tar.gz <!-- BEGIN SHA linux-arm64_noruntime --><LINUX_ARM64_SHA_NORUNTIME><!-- END SHA linux-arm64_noruntime -->
-- actions-runner-linux-arm-<RUNNER_VERSION>-noruntime.tar.gz <!-- BEGIN SHA linux-arm_noruntime --><LINUX_ARM_SHA_NORUNTIME><!-- END SHA linux-arm_noruntime -->
-
-- actions-runner-win-x64-<RUNNER_VERSION>-noruntime-noexternals.zip <!-- BEGIN SHA win-x64_noruntime_noexternals --><WIN_X64_SHA_NORUNTIME_NOEXTERNALS><!-- END SHA win-x64_noruntime_noexternals -->
-- actions-runner-win-arm64-<RUNNER_VERSION>-noruntime-noexternals.zip <!-- BEGIN SHA win-arm64_noruntime_noexternals --><WIN_ARM64_SHA_NORUNTIME_NOEXTERNALS><!-- END SHA win-arm64_noruntime_noexternals -->
-- actions-runner-osx-x64-<RUNNER_VERSION>-noruntime-noexternals.tar.gz <!-- BEGIN SHA osx-x64_noruntime_noexternals --><OSX_X64_SHA_NORUNTIME_NOEXTERNALS><!-- END SHA osx-x64_noruntime_noexternals -->
-- actions-runner-osx-arm64-<RUNNER_VERSION>-noruntime-noexternals.tar.gz <!-- BEGIN SHA osx-arm64_noruntime_noexternals --><OSX_ARM64_SHA_NORUNTIME_NOEXTERNALS><!-- END SHA osx-arm64_noruntime_noexternals -->
-- actions-runner-linux-x64-<RUNNER_VERSION>-noruntime-noexternals.tar.gz <!-- BEGIN SHA linux-x64_noruntime_noexternals --><LINUX_X64_SHA_NORUNTIME_NOEXTERNALS><!-- END SHA linux-x64_noruntime_noexternals -->
-- actions-runner-linux-arm64-<RUNNER_VERSION>-noruntime-noexternals.tar.gz <!-- BEGIN SHA linux-arm64_noruntime_noexternals --><LINUX_ARM64_SHA_NORUNTIME_NOEXTERNALS><!-- END SHA linux-arm64_noruntime_noexternals -->
-- actions-runner-linux-arm-<RUNNER_VERSION>-noruntime-noexternals.tar.gz <!-- BEGIN SHA linux-arm_noruntime_noexternals --><LINUX_ARM_SHA_NORUNTIME_NOEXTERNALS><!-- END SHA linux-arm_noruntime_noexternals -->

releaseVersion

@@ -1 +1 @@
-<Update to ./src/runnerversion when creating release>
+2.320.0

src/Misc/contentHash/dotnetRuntime/linux-arm

@@ -1 +0,0 @@
-531b31914e525ecb12cc5526415bc70a112ebc818f877347af1a231011f539c5

src/Misc/contentHash/dotnetRuntime/linux-arm64

@@ -1 +0,0 @@
-722dd5fa5ecc207fcccf67f6e502d689f2119d8117beff2041618fba17dc66a4

src/Misc/contentHash/dotnetRuntime/linux-x64

@@ -1 +0,0 @@
-8ca75c76e15ab9dc7fe49a66c5c74e171e7fabd5d26546fda8931bd11bff30f9

src/Misc/contentHash/dotnetRuntime/osx-arm64

@@ -1 +0,0 @@
-70496eb1c99b39b3373b5088c95a35ebbaac1098e6c47c8aab94771f3ffbf501

src/Misc/contentHash/dotnetRuntime/osx-x64

@@ -1 +0,0 @@
-4f8d48727d535daabcaec814e0dafb271c10625366c78e7e022ca7477a73023f

src/Misc/contentHash/dotnetRuntime/win-arm64

@@ -1 +0,0 @@
-d54d7428f2b9200a0030365a6a4e174e30a1b29b922f8254dffb2924bd09549d

src/Misc/contentHash/dotnetRuntime/win-x64

@@ -1 +0,0 @@
-eaa939c45307f46b7003902255b3a2a09287215d710984107667e03ac493eb26

src/Misc/contentHash/externals/linux-arm

@@ -1 +0,0 @@
-4bf3e1af0d482af1b2eaf9f08250248a8c1aea8ec20a3c5be116d58cdd930009

src/Misc/contentHash/externals/linux-arm64

@@ -1 +0,0 @@
-ec1719a8cb4d8687328aa64f4aa7c4e3498a715d8939117874782e3e6e63a14b

src/Misc/contentHash/externals/linux-x64

@@ -1 +0,0 @@
-50538de29f173bb73f708c4ed2c8328a62b8795829b97b2a6cb57197e2305287

src/Misc/contentHash/externals/osx-arm64

@@ -1 +0,0 @@
-a0a96cbb7593643b69e669bf14d7b29b7f27800b3a00bb3305aebe041456c701

src/Misc/contentHash/externals/osx-x64

@@ -1 +0,0 @@
-6255b22692779467047ecebd60ad46984866d75cdfe10421d593a7b51d620b09

src/Misc/contentHash/externals/win-arm64

@@ -1 +0,0 @@
-6ff1abd055dc35bfbf06f75c2f08908f660346f66ad1d8f81c910068e9ba029d

src/Misc/contentHash/externals/win-x64

@@ -1 +0,0 @@
-433a6d748742d12abd20dc2a79b62ac3d9718ae47ef26f8e84dc8c180eea3659

src/Misc/externals.sh

@@ -5,10 +5,11 @@ PRECACHE=$2
 NODE_URL=https://nodejs.org/dist
 UNOFFICIAL_NODE_URL=https://unofficial-builds.nodejs.org/download/release
 NODE_ALPINE_URL=https://github.com/actions/alpine_nodejs/releases/download
+# When you update Node versions you must also create a new release of alpine_nodejs at that updated version.
+# Follow the instructions here: https://github.com/actions/alpine_nodejs?tab=readme-ov-file#getting-started
 NODE16_VERSION="16.20.2"
-NODE20_VERSION="20.8.1"
-# used only for win-arm64, remove node16 unofficial version when official version is available
-NODE16_UNOFFICIAL_VERSION="16.20.0"
+NODE20_VERSION="20.13.1"
+NODE16_UNOFFICIAL_VERSION="16.20.0" # used only for win-arm64, remove node16 unofficial version when official version is available
 
 get_abs_path() {
   # exploits the fact that pwd will print abs path when no args
@@ -63,17 +64,16 @@ function acquireExternalTool() {
             echo "Curl version: $CURL_VERSION"
 
             # curl -f Fail silently (no output at all) on HTTP errors (H)
-            #      -k Allow connections to SSL sites without certs (H)
             #      -S Show error. With -s, make curl show errors when they occur
             #      -L Follow redirects (H)
             #      -o FILE    Write to FILE instead of stdout
             #      --retry 3   Retries transient errors 3 times (timeouts, 5xx)
             if [[ "$(printf '%s\n' "7.71.0" "$CURL_VERSION" | sort -V | head -n1)" != "7.71.0" ]]; then
                 # Curl version is less than or equal to 7.71.0, skipping retry-all-errors flag
-                 curl -fkSL --retry 3 -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl'
+                 curl -fSL --retry 3 -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl'
             else
                 # Curl version is greater than 7.71.0, running curl with --retry-all-errors flag
-                 curl -fkSL --retry 3 --retry-all-errors -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl'
+                 curl -fSL --retry 3 --retry-all-errors -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl'
             fi
 
             # Move the partial file to the download target.

src/Misc/layoutbin/RunnerService.js

@@ -114,6 +114,11 @@ var runService = function () {
             );
             stopping = true;
           }
+        } else if (code === 5) {
+          console.log(
+            "Runner listener exit with Session Conflict error, stop the service, no retry needed."
+          );
+          stopping = true;
         } else {
           var messagePrefix = "Runner listener exit with undefined return code";
           unknownFailureRetryCount++;

src/Misc/layoutroot/run-helper.cmd.template

@@ -49,5 +49,10 @@ if %ERRORLEVEL% EQU 4 (
   exit /b 1
 )
 
+if %ERRORLEVEL% EQU 5 (
+  echo "Runner listener exit with Session Conflict error, stop the service, no retry needed."
+  exit /b 0
+)
+
 echo "Exiting after unknown error code: %ERRORLEVEL%"
 exit /b 0

src/Misc/layoutroot/run-helper.sh.template

@@ -70,6 +70,9 @@ elif [[ $returnCode == 4 ]]; then
         "$DIR"/safe_sleep.sh 1
     done
     exit 2
+elif [[ $returnCode == 5 ]]; then
+    echo "Runner listener exit with Session Conflict error, stop the service, no retry needed."
+    exit 0
 else
     echo "Exiting with unknown error code: ${returnCode}"
     exit 0

src/Misc/layoutroot/run.sh

@@ -38,7 +38,7 @@ runWithManualTrap() {
         cp -f "$DIR"/run-helper.sh.template "$DIR"/run-helper.sh
         "$DIR"/run-helper.sh $* &
         PID=$!
-        wait -f $PID
+        wait $PID
         returnCode=$?
         if [[ $returnCode -eq 2 ]]; then
             echo "Restarting runner..."
@@ -84,4 +84,4 @@ if [[ -z "$RUNNER_MANUALLY_TRAP_SIG" ]]; then
     run $*
 else
     runWithManualTrap $*
-fi
+fi

src/Misc/runnercoreassets

@@ -1,57 +0,0 @@
-actions.runner.plist.template
-actions.runner.service.template
-checkScripts/downloadCert.js
-checkScripts/makeWebRequest.js
-darwin.svc.sh.template
-hashFiles/index.js
-installdependencies.sh
-macos-run-invoker.js
-Microsoft.IdentityModel.Logging.dll
-Microsoft.IdentityModel.Tokens.dll
-Minimatch.dll
-Newtonsoft.Json.Bson.dll
-Newtonsoft.Json.dll
-Runner.Common.deps.json
-Runner.Common.dll
-Runner.Common.pdb
-Runner.Listener
-Runner.Listener.deps.json
-Runner.Listener.dll
-Runner.Listener.exe
-Runner.Listener.pdb
-Runner.Listener.runtimeconfig.json
-Runner.PluginHost
-Runner.PluginHost.deps.json
-Runner.PluginHost.dll
-Runner.PluginHost.exe
-Runner.PluginHost.pdb
-Runner.PluginHost.runtimeconfig.json
-Runner.Plugins.deps.json
-Runner.Plugins.dll
-Runner.Plugins.pdb
-Runner.Sdk.deps.json
-Runner.Sdk.dll
-Runner.Sdk.pdb
-Runner.Worker
-Runner.Worker.deps.json
-Runner.Worker.dll
-Runner.Worker.exe
-Runner.Worker.pdb
-Runner.Worker.runtimeconfig.json
-RunnerService.exe
-RunnerService.exe.config
-RunnerService.js
-RunnerService.pdb
-runsvc.sh
-Sdk.deps.json
-Sdk.dll
-Sdk.pdb
-System.IdentityModel.Tokens.Jwt.dll
-System.Net.Http.Formatting.dll
-System.Security.Cryptography.Pkcs.dll
-System.Security.Cryptography.ProtectedData.dll
-System.ServiceProcess.ServiceController.dll
-systemd.svc.sh.template
-update.cmd.template
-update.sh.template
-YamlDotNet.dll

src/Misc/runnerdotnetruntimeassets

@@ -1,270 +0,0 @@
-api-ms-win-core-console-l1-1-0.dll
-api-ms-win-core-console-l1-2-0.dll
-api-ms-win-core-datetime-l1-1-0.dll
-api-ms-win-core-debug-l1-1-0.dll
-api-ms-win-core-errorhandling-l1-1-0.dll
-api-ms-win-core-fibers-l1-1-0.dll
-api-ms-win-core-file-l1-1-0.dll
-api-ms-win-core-file-l1-2-0.dll
-api-ms-win-core-file-l2-1-0.dll
-api-ms-win-core-handle-l1-1-0.dll
-api-ms-win-core-heap-l1-1-0.dll
-api-ms-win-core-interlocked-l1-1-0.dll
-api-ms-win-core-libraryloader-l1-1-0.dll
-api-ms-win-core-localization-l1-2-0.dll
-api-ms-win-core-memory-l1-1-0.dll
-api-ms-win-core-namedpipe-l1-1-0.dll
-api-ms-win-core-processenvironment-l1-1-0.dll
-api-ms-win-core-processthreads-l1-1-0.dll
-api-ms-win-core-processthreads-l1-1-1.dll
-api-ms-win-core-profile-l1-1-0.dll
-api-ms-win-core-rtlsupport-l1-1-0.dll
-api-ms-win-core-string-l1-1-0.dll
-api-ms-win-core-synch-l1-1-0.dll
-api-ms-win-core-synch-l1-2-0.dll
-api-ms-win-core-sysinfo-l1-1-0.dll
-api-ms-win-core-timezone-l1-1-0.dll
-api-ms-win-core-util-l1-1-0.dll
-api-ms-win-crt-conio-l1-1-0.dll
-api-ms-win-crt-convert-l1-1-0.dll
-api-ms-win-crt-environment-l1-1-0.dll
-api-ms-win-crt-filesystem-l1-1-0.dll
-api-ms-win-crt-heap-l1-1-0.dll
-api-ms-win-crt-locale-l1-1-0.dll
-api-ms-win-crt-math-l1-1-0.dll
-api-ms-win-crt-multibyte-l1-1-0.dll
-api-ms-win-crt-private-l1-1-0.dll
-api-ms-win-crt-process-l1-1-0.dll
-api-ms-win-crt-runtime-l1-1-0.dll
-api-ms-win-crt-stdio-l1-1-0.dll
-api-ms-win-crt-string-l1-1-0.dll
-api-ms-win-crt-time-l1-1-0.dll
-api-ms-win-crt-utility-l1-1-0.dll
-clrcompression.dll
-clretwrc.dll
-clrjit.dll
-coreclr.dll
-createdump
-createdump.exe
-dbgshim.dll
-hostfxr.dll
-hostpolicy.dll
-libclrjit.dylib
-libclrjit.so
-libcoreclr.dylib
-libcoreclr.so
-libcoreclrtraceptprovider.so
-libdbgshim.dylib
-libdbgshim.so
-libhostfxr.dylib
-libhostfxr.so
-libhostpolicy.dylib
-libhostpolicy.so
-libmscordaccore.dylib
-libmscordaccore.so
-libmscordbi.dylib
-libmscordbi.so
-Microsoft.CSharp.dll
-Microsoft.DiaSymReader.Native.amd64.dll
-Microsoft.DiaSymReader.Native.arm64.dll
-Microsoft.VisualBasic.Core.dll
-Microsoft.VisualBasic.dll
-Microsoft.Win32.Primitives.dll
-Microsoft.Win32.Registry.dll
-mscordaccore.dll
-mscordaccore_amd64_amd64_6.0.522.21309.dll
-mscordaccore_arm64_arm64_6.0.522.21309.dll
-mscordaccore_amd64_amd64_6.0.1322.58009.dll
-mscordaccore_amd64_amd64_6.0.2023.32017.dll
-mscordaccore_amd64_amd64_6.0.2223.42425.dll
-mscordaccore_amd64_amd64_6.0.2323.48002.dll
-mscordbi.dll
-mscorlib.dll
-mscorrc.debug.dll
-mscorrc.dll
-msquic.dll
-netstandard.dll
-SOS_README.md
-System.AppContext.dll
-System.Buffers.dll
-System.Collections.Concurrent.dll
-System.Collections.dll
-System.Collections.Immutable.dll
-System.Collections.NonGeneric.dll
-System.Collections.Specialized.dll
-System.ComponentModel.Annotations.dll
-System.ComponentModel.DataAnnotations.dll
-System.ComponentModel.dll
-System.ComponentModel.EventBasedAsync.dll
-System.ComponentModel.Primitives.dll
-System.ComponentModel.TypeConverter.dll
-System.Configuration.dll
-System.Console.dll
-System.Core.dll
-System.Data.Common.dll
-System.Data.DataSetExtensions.dll
-System.Data.dll
-System.Diagnostics.Contracts.dll
-System.Diagnostics.Debug.dll
-System.Diagnostics.DiagnosticSource.dll
-System.Diagnostics.FileVersionInfo.dll
-System.Diagnostics.Process.dll
-System.Diagnostics.StackTrace.dll
-System.Diagnostics.TextWriterTraceListener.dll
-System.Diagnostics.Tools.dll
-System.Diagnostics.TraceSource.dll
-System.Diagnostics.Tracing.dll
-System.dll
-System.Drawing.dll
-System.Drawing.Primitives.dll
-System.Dynamic.Runtime.dll
-System.Formats.Asn1.dll
-System.Globalization.Calendars.dll
-System.Globalization.dll
-System.Globalization.Extensions.dll
-System.Globalization.Native.dylib
-System.Globalization.Native.so
-System.IO.Compression.Brotli.dll
-System.IO.Compression.dll
-System.IO.Compression.FileSystem.dll
-System.IO.Compression.Native.a
-System.IO.Compression.Native.dll
-System.IO.Compression.Native.dylib
-System.IO.Compression.Native.so
-System.IO.Compression.ZipFile.dll
-System.IO.dll
-System.IO.FileSystem.AccessControl.dll
-System.IO.FileSystem.dll
-System.IO.FileSystem.DriveInfo.dll
-System.IO.FileSystem.Primitives.dll
-System.IO.FileSystem.Watcher.dll
-System.IO.IsolatedStorage.dll
-System.IO.MemoryMappedFiles.dll
-System.IO.Pipes.AccessControl.dll
-System.IO.Pipes.dll
-System.IO.UnmanagedMemoryStream.dll
-System.Linq.dll
-System.Linq.Expressions.dll
-System.Linq.Parallel.dll
-System.Linq.Queryable.dll
-System.Memory.dll
-System.Native.a
-System.Native.dylib
-System.Native.so
-System.Net.dll
-System.Net.Http.dll
-System.Net.Http.Json.dll
-System.Net.Http.Native.a
-System.Net.Http.Native.dylib
-System.Net.Http.Native.so
-System.Net.HttpListener.dll
-System.Net.Mail.dll
-System.Net.NameResolution.dll
-System.Net.NetworkInformation.dll
-System.Net.Ping.dll
-System.Net.Primitives.dll
-System.Net.Quic.dll
-System.Net.Requests.dll
-System.Net.Security.dll
-System.Net.Security.Native.a
-System.Net.Security.Native.dylib
-System.Net.Security.Native.so
-System.Net.ServicePoint.dll
-System.Net.Sockets.dll
-System.Net.WebClient.dll
-System.Net.WebHeaderCollection.dll
-System.Net.WebProxy.dll
-System.Net.WebSockets.Client.dll
-System.Net.WebSockets.dll
-System.Numerics.dll
-System.Numerics.Vectors.dll
-System.ObjectModel.dll
-System.Private.CoreLib.dll
-System.Private.DataContractSerialization.dll
-System.Private.Uri.dll
-System.Private.Xml.dll
-System.Private.Xml.Linq.dll
-System.Reflection.DispatchProxy.dll
-System.Reflection.dll
-System.Reflection.Emit.dll
-System.Reflection.Emit.ILGeneration.dll
-System.Reflection.Emit.Lightweight.dll
-System.Reflection.Extensions.dll
-System.Reflection.Metadata.dll
-System.Reflection.Primitives.dll
-System.Reflection.TypeExtensions.dll
-System.Resources.Reader.dll
-System.Resources.ResourceManager.dll
-System.Resources.Writer.dll
-System.Runtime.CompilerServices.Unsafe.dll
-System.Runtime.CompilerServices.VisualC.dll
-System.Runtime.dll
-System.Runtime.Extensions.dll
-System.Runtime.Handles.dll
-System.Runtime.InteropServices.dll
-System.Runtime.InteropServices.RuntimeInformation.dll
-System.Runtime.InteropServices.WindowsRuntime.dll
-System.Runtime.Intrinsics.dll
-System.Runtime.Loader.dll
-System.Runtime.Numerics.dll
-System.Runtime.Serialization.dll
-System.Runtime.Serialization.Formatters.dll
-System.Runtime.Serialization.Json.dll
-System.Runtime.Serialization.Primitives.dll
-System.Runtime.Serialization.Xml.dll
-System.Runtime.WindowsRuntime.dll
-System.Runtime.WindowsRuntime.UI.Xaml.dll
-System.Security.AccessControl.dll
-System.Security.Claims.dll
-System.Security.Cryptography.Algorithms.dll
-System.Security.Cryptography.Cng.dll
-System.Security.Cryptography.Csp.dll
-System.Security.Cryptography.Encoding.dll
-System.Security.Cryptography.Native.Apple.a
-System.Security.Cryptography.Native.Apple.dylib
-System.Security.Cryptography.Native.OpenSsl.a
-System.Security.Cryptography.Native.OpenSsl.dylib
-System.Security.Cryptography.Native.OpenSsl.so
-System.Security.Cryptography.OpenSsl.dll
-System.Security.Cryptography.Primitives.dll
-System.Security.Cryptography.X509Certificates.dll
-System.Security.Cryptography.XCertificates.dll
-System.Security.dll
-System.Security.Principal.dll
-System.Security.Principal.Windows.dll
-System.Security.SecureString.dll
-System.ServiceModel.Web.dll
-System.ServiceProcess.dll
-System.Text.Encoding.CodePages.dll
-System.Text.Encoding.dll
-System.Text.Encoding.Extensions.dll
-System.Text.Encodings.Web.dll
-System.Text.Json.dll
-System.Text.RegularExpressions.dll
-System.Threading.Channels.dll
-System.Threading.dll
-System.Threading.Overlapped.dll
-System.Threading.Tasks.Dataflow.dll
-System.Threading.Tasks.dll
-System.Threading.Tasks.Extensions.dll
-System.Threading.Tasks.Parallel.dll
-System.Threading.Thread.dll
-System.Threading.ThreadPool.dll
-System.Threading.Timer.dll
-System.Transactions.dll
-System.Transactions.Local.dll
-System.ValueTuple.dll
-System.Web.dll
-System.Web.HttpUtility.dll
-System.Windows.dll
-System.Xml.dll
-System.Xml.Linq.dll
-System.Xml.ReaderWriter.dll
-System.Xml.Serialization.dll
-System.Xml.XDocument.dll
-System.Xml.XmlDocument.dll
-System.Xml.XmlSerializer.dll
-System.Xml.XPath.dll
-System.Xml.XPath.XDocument.dll
-ucrtbase.dll
-WindowsBase.dll

src/Misc/trimmedpackages_targz.json

@@ -1,24 +0,0 @@
-[
-    {
-        "HashValue": "<NO_RUNTIME_EXTERNALS_HASH>",
-        "DownloadUrl": "https://github.com/actions/runner/releases/download/v<RUNNER_VERSION>/actions-runner-<RUNNER_PLATFORM>-<RUNNER_VERSION>-noruntime-noexternals.tar.gz",
-        "TrimmedContents": {
-            "dotnetRuntime": "<RUNTIME_HASH>",
-            "externals": "<EXTERNALS_HASH>"
-        }
-    },
-    {
-        "HashValue": "<NO_RUNTIME_HASH>",
-        "DownloadUrl": "https://github.com/actions/runner/releases/download/v<RUNNER_VERSION>/actions-runner-<RUNNER_PLATFORM>-<RUNNER_VERSION>-noruntime.tar.gz",
-        "TrimmedContents": {
-            "dotnetRuntime": "<RUNTIME_HASH>"
-        }
-    },
-    {
-        "HashValue": "<NO_EXTERNALS_HASH>",
-        "DownloadUrl": "https://github.com/actions/runner/releases/download/v<RUNNER_VERSION>/actions-runner-<RUNNER_PLATFORM>-<RUNNER_VERSION>-noexternals.tar.gz",
-        "TrimmedContents": {
-            "externals": "<EXTERNALS_HASH>"
-        }
-    }
-]

src/Misc/trimmedpackages_zip.json

@@ -1,24 +0,0 @@
-[
-    {
-        "HashValue": "<NO_RUNTIME_EXTERNALS_HASH>",
-        "DownloadUrl": "https://github.com/actions/runner/releases/download/v<RUNNER_VERSION>/actions-runner-<RUNNER_PLATFORM>-<RUNNER_VERSION>-noruntime-noexternals.zip",
-        "TrimmedContents": {
-            "dotnetRuntime": "<RUNTIME_HASH>",
-            "externals": "<EXTERNALS_HASH>"
-        }
-    },
-    {
-        "HashValue": "<NO_RUNTIME_HASH>",
-        "DownloadUrl": "https://github.com/actions/runner/releases/download/v<RUNNER_VERSION>/actions-runner-<RUNNER_PLATFORM>-<RUNNER_VERSION>-noruntime.zip",
-        "TrimmedContents": {
-            "dotnetRuntime": "<RUNTIME_HASH>"
-        }
-    },
-    {
-        "HashValue": "<NO_EXTERNALS_HASH>",
-        "DownloadUrl": "https://github.com/actions/runner/releases/download/v<RUNNER_VERSION>/actions-runner-<RUNNER_PLATFORM>-<RUNNER_VERSION>-noexternals.zip",
-        "TrimmedContents": {
-            "externals": "<EXTERNALS_HASH>"
-        }
-    }
-]

src/Runner.Common/ActionsRunServer.cs

@@ -20,12 +20,12 @@ namespace GitHub.Runner.Common
     {
         private bool _hasConnection;
         private VssConnection _connection;
-        private TaskAgentHttpClient _taskAgentClient;
+        private ActionsRunServerHttpClient _actionsRunServerClient;
 
         public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials)
         {
             _connection = await EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
-            _taskAgentClient = _connection.GetClient<TaskAgentHttpClient>();
+            _actionsRunServerClient = _connection.GetClient<ActionsRunServerHttpClient>();
             _hasConnection = true;
         }
 
@@ -42,7 +42,7 @@ namespace GitHub.Runner.Common
             CheckConnection();
             var jobMessage = RetryRequest<AgentJobRequestMessage>(async () =>
                                                     {
-                                                        return await _taskAgentClient.GetJobMessageAsync(id, cancellationToken);
+                                                        return await _actionsRunServerClient.GetJobMessageAsync(id, cancellationToken);
                                                     }, cancellationToken);
 
             return jobMessage;

src/Runner.Common/BrokerServer.cs

@@ -17,7 +17,14 @@ namespace GitHub.Runner.Common
     {
         Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
 
-        Task<TaskAgentMessage> GetRunnerMessageAsync(CancellationToken token, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate);
+        Task<TaskAgentSession> CreateSessionAsync(TaskAgentSession session, CancellationToken cancellationToken);
+        Task DeleteSessionAsync(CancellationToken cancellationToken);
+
+        Task<TaskAgentMessage> GetRunnerMessageAsync(Guid? sessionId, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate, CancellationToken token);
+
+        Task UpdateConnectionIfNeeded(Uri serverUri, VssCredentials credentials);
+
+        Task ForceRefreshConnection(VssCredentials credentials);
     }
 
     public sealed class BrokerServer : RunnerService, IBrokerServer
@@ -44,13 +51,53 @@ namespace GitHub.Runner.Common
             }
         }
 
-        public Task<TaskAgentMessage> GetRunnerMessageAsync(CancellationToken cancellationToken, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate)
+        public async Task<TaskAgentSession> CreateSessionAsync(TaskAgentSession session, CancellationToken cancellationToken)
         {
             CheckConnection();
-            var jobMessage = RetryRequest<TaskAgentMessage>(
-                async () => await _brokerHttpClient.GetRunnerMessageAsync(version, status, os, architecture, disableUpdate, cancellationToken), cancellationToken);
+            var jobMessage = await _brokerHttpClient.CreateSessionAsync(session, cancellationToken);
 
             return jobMessage;
         }
+
+        public Task<TaskAgentMessage> GetRunnerMessageAsync(Guid? sessionId, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate, CancellationToken cancellationToken)
+        {
+            CheckConnection();
+            var brokerSession = RetryRequest<TaskAgentMessage>(
+                async () => await _brokerHttpClient.GetRunnerMessageAsync(sessionId, version, status, os, architecture, disableUpdate, cancellationToken), cancellationToken, shouldRetry: ShouldRetryException);
+
+
+            return brokerSession;
+        }
+
+        public async Task DeleteSessionAsync(CancellationToken cancellationToken)
+        {
+            CheckConnection();
+            await _brokerHttpClient.DeleteSessionAsync(cancellationToken);
+        }
+
+        public Task UpdateConnectionIfNeeded(Uri serverUri, VssCredentials credentials)
+        {
+            if (_brokerUri != serverUri || !_hasConnection)
+            {
+                return ConnectAsync(serverUri, credentials);
+            }
+
+            return Task.CompletedTask;
+        }
+
+        public Task ForceRefreshConnection(VssCredentials credentials)
+        {
+            return ConnectAsync(_brokerUri, credentials);
+        }
+
+        public bool ShouldRetryException(Exception ex)
+        {
+            if (ex is AccessDeniedException ade)
+            {
+                return false;
+            }
+
+            return true;
+        }
     }
 }

src/Runner.Common/Constants.cs

@@ -153,6 +153,7 @@ namespace GitHub.Runner.Common
                 public const int RetryableError = 2;
                 public const int RunnerUpdating = 3;
                 public const int RunOnceRunnerUpdating = 4;
+                public const int SessionConflict = 5;
             }
 
             public static class Features
@@ -180,6 +181,9 @@ namespace GitHub.Runner.Common
             public static readonly string DeprecatedNodeVersion = "node16";
             public static readonly string EnforcedNode12DetectedAfterEndOfLife = "The following actions uses node12 which is deprecated and will be forced to run on node16: {0}. For more info: https://github.blog/changelog/2023-06-13-github-actions-all-actions-will-run-on-node16-instead-of-node12-by-default/";
             public static readonly string EnforcedNode12DetectedAfterEndOfLifeEnvVariable = "Node16ForceActionsWarnings";
+            public static readonly string EnforcedNode16DetectedAfterEndOfLife = "The following actions use a deprecated Node.js version and will be forced to run on node20: {0}. For more info: https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/";
+            public static readonly string EnforcedNode16DetectedAfterEndOfLifeEnvVariable = "Node20ForceActionsWarnings";
+
         }
 
         public static class RunnerEvent
@@ -251,6 +255,7 @@ namespace GitHub.Runner.Common
                 public static readonly string RunnerDebug = "ACTIONS_RUNNER_DEBUG";
                 public static readonly string StepDebug = "ACTIONS_STEP_DEBUG";
                 public static readonly string AllowActionsUseUnsecureNodeVersion = "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION";
+                public static readonly string ManualForceActionsToNode20 = "FORCE_JAVASCRIPT_ACTIONS_TO_NODE20";
             }
 
             public static class Agent
@@ -262,6 +267,7 @@ namespace GitHub.Runner.Common
                 public static readonly string ForcedActionsNodeVersion = "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION";
                 public static readonly string PrintLogToStdout = "ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT";
                 public static readonly string ActionArchiveCacheDirectory = "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE";
+                public static readonly string ManualForceActionsToNode20 = "FORCE_JAVASCRIPT_ACTIONS_TO_NODE20";
             }
 
             public static class System
@@ -274,6 +280,10 @@ namespace GitHub.Runner.Common
                 public static readonly string PhaseDisplayName = "system.phaseDisplayName";
                 public static readonly string JobRequestType = "system.jobRequestType";
                 public static readonly string OrchestrationId = "system.orchestrationId";
+                public static readonly string TestDotNet8Compatibility = "system.testDotNet8Compatibility";
+                public static readonly string DotNet8CompatibilityOutputLength = "system.dotNet8CompatibilityOutputLength";
+                public static readonly string DotNet8CompatibilityOutputPattern = "system.dotNet8CompatibilityOutputPattern";
+                public static readonly string DotNet8CompatibilityWarning = "system.dotNet8CompatibilityWarning";
             }
         }
 

src/Runner.Common/HostContext.cs

@@ -36,6 +36,7 @@ namespace GitHub.Runner.Common
         event EventHandler Unloading;
         void ShutdownRunner(ShutdownReason reason);
         void WritePerfCounter(string counter);
+        void LoadDefaultUserAgents();
     }
 
     public enum StartupType
@@ -67,6 +68,7 @@ namespace GitHub.Runner.Common
         private StartupType _startupType;
         private string _perfFile;
         private RunnerWebProxy _webProxy = new();
+        private string _hostType = string.Empty;
 
         public event EventHandler Unloading;
         public CancellationToken RunnerShutdownToken => _runnerShutdownTokenSource.Token;
@@ -78,6 +80,7 @@ namespace GitHub.Runner.Common
         {
             // Validate args.
             ArgUtil.NotNullOrEmpty(hostType, nameof(hostType));
+            _hostType = hostType;
 
             _loadContext = AssemblyLoadContext.GetLoadContext(typeof(HostContext).GetTypeInfo().Assembly);
             _loadContext.Unloading += LoadContext_Unloading;
@@ -196,6 +199,16 @@ namespace GitHub.Runner.Common
                 }
             }
 
+            if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY")))
+            {
+                _trace.Warning($"Runner is running under insecure mode: HTTPS server certificate validation has been turned off by GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY environment variable.");
+            }
+
+            LoadDefaultUserAgents();
+        }
+
+        public void LoadDefaultUserAgents()
+        {
             if (string.IsNullOrEmpty(WebProxy.HttpProxyAddress) && string.IsNullOrEmpty(WebProxy.HttpsProxyAddress))
             {
                 _trace.Info($"No proxy settings were found based on environmental variables (http_proxy/https_proxy/HTTP_PROXY/HTTPS_PROXY)");
@@ -205,11 +218,6 @@ namespace GitHub.Runner.Common
                 _userAgents.Add(new ProductInfoHeaderValue("HttpProxyConfigured", bool.TrueString));
             }
 
-            if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY")))
-            {
-                _trace.Warning($"Runner is running under insecure mode: HTTPS server certificate validation has been turned off by GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY environment variable.");
-            }
-
             var credFile = GetConfigFile(WellKnownConfigFile.Credentials);
             if (File.Exists(credFile))
             {
@@ -244,6 +252,11 @@ namespace GitHub.Runner.Common
                 _trace.Info($"Adding extra user agent '{extraUserAgentHeader}' to all HTTP requests.");
                 _userAgents.Add(extraUserAgentHeader);
             }
+
+            var currentProcess = Process.GetCurrentProcess();
+            _userAgents.Add(new ProductInfoHeaderValue("Pid", currentProcess.Id.ToString()));
+            _userAgents.Add(new ProductInfoHeaderValue("CreationTime", Uri.EscapeDataString(DateTime.UtcNow.ToString("O"))));
+            _userAgents.Add(new ProductInfoHeaderValue($"({_hostType})"));
         }
 
         public string GetDirectory(WellKnownDirectory directory)

src/Runner.Common/JobServer.cs

@@ -4,6 +4,7 @@ using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Headers;
+using System.Net.Security;
 using System.Net.WebSockets;
 using System.Text;
 using System.Threading;
@@ -179,6 +180,10 @@ namespace GitHub.Runner.Common
                     userAgentValues.AddRange(UserAgentUtility.GetDefaultRestUserAgent());
                     userAgentValues.AddRange(HostContext.UserAgents);
                     this._websocketClient.Options.SetRequestHeader("User-Agent", string.Join(" ", userAgentValues.Select(x => x.ToString())));
+                    if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY")))
+                    {
+                        this._websocketClient.Options.RemoteCertificateValidationCallback = (_, _, _, _) => true;
+                    }
 
                     this._websocketConnectTask = ConnectWebSocketClient(feedStreamUrl, delay);
                 }

src/Runner.Common/JobServerQueue.cs

@@ -19,7 +19,7 @@ namespace GitHub.Runner.Common
         TaskCompletionSource<int> JobRecordUpdated { get; }
         event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling;
         Task ShutdownAsync();
-        void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false, bool enableTelemetry = false);
+        void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false);
         void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null);
         void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource);
         void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines);
@@ -74,6 +74,7 @@ namespace GitHub.Runner.Common
         private readonly List<JobTelemetry> _jobTelemetries = new();
         private bool _queueInProcess = false;
         private bool _resultsServiceOnly = false;
+        private int _resultsServiceExceptionsCount = 0;
         private Stopwatch _resultsUploadTimer = new();
         private Stopwatch _actionsUploadTimer = new();
 
@@ -104,11 +105,10 @@ namespace GitHub.Runner.Common
             _resultsServer = hostContext.GetService<IResultsServer>();
         }
 
-        public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false, bool enableTelemetry = false)
+        public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false)
         {
             Trace.Entering();
             _resultsServiceOnly = resultsServiceOnly;
-            _enableTelemetry = enableTelemetry;
 
             var serviceEndPoint = jobRequest.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
 
@@ -134,11 +134,17 @@ namespace GitHub.Runner.Common
                 {
                     liveConsoleFeedUrl = feedStreamUrl;
                 }
-
-                _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken);
+                jobRequest.Variables.TryGetValue("system.github.results_upload_with_sdk", out VariableValue resultsUseSdkVariable);
+                _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken, StringUtil.ConvertToBoolean(resultsUseSdkVariable?.Value));
                 _resultsClientInitiated = true;
             }
 
+            // Enable telemetry if we have both results service and actions service
+            if (_resultsClientInitiated && !_resultsServiceOnly)
+            {
+                _enableTelemetry = true;
+            }
+
             if (_queueInProcess)
             {
                 Trace.Info("No-opt, all queue process tasks are running.");
@@ -551,6 +557,10 @@ namespace GitHub.Runner.Common
                             {
                                 await UploadSummaryFile(file);
                             }
+                            if (string.Equals(file.Type, CoreAttachmentType.ResultsDiagnosticLog, StringComparison.OrdinalIgnoreCase))
+                            {
+                                await UploadResultsDiagnosticLogsFile(file);
+                            }
                             else if (String.Equals(file.Type, CoreAttachmentType.ResultsLog, StringComparison.OrdinalIgnoreCase))
                             {
                                 if (file.RecordId != _jobTimelineRecordId)
@@ -570,9 +580,9 @@ namespace GitHub.Runner.Common
                             Trace.Info("Catch exception during file upload to results, keep going since the process is best effort.");
                             Trace.Error(ex);
                             errorCount++;
-
+                            _resultsServiceExceptionsCount++;
                             // If we hit any exceptions uploading to Results, let's skip any additional uploads to Results unless Results is serving logs
-                            if (!_resultsServiceOnly)
+                            if (!_resultsServiceOnly && _resultsServiceExceptionsCount > 3)
                             {
                                 _resultsClientInitiated = false;
                                 SendResultsTelemetry(ex);
@@ -603,7 +613,7 @@ namespace GitHub.Runner.Common
 
         private void SendResultsTelemetry(Exception ex)
         {
-            var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception with results. {ex.Message}" };
+            var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception with results. {HostContext.SecretMasker.MaskSecrets(ex.Message)}" };
             issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.ResultsUploadFailure;
 
             var telemetryRecord = new TimelineRecord()
@@ -699,7 +709,9 @@ namespace GitHub.Runner.Common
                             {
                                 Trace.Info("Catch exception during update steps, skip update Results.");
                                 Trace.Error(e);
-                                if (!_resultsServiceOnly)
+                                _resultsServiceExceptionsCount++;
+                                // If we hit any exceptions uploading to Results, let's skip any additional uploads to Results unless Results is serving logs
+                                if (!_resultsServiceOnly && _resultsServiceExceptionsCount > 3)
                                 {
                                     _resultsClientInitiated = false;
                                     SendResultsTelemetry(e);
@@ -922,6 +934,17 @@ namespace GitHub.Runner.Common
             await UploadResultsFile(file, summaryHandler);
         }
 
+        private async Task UploadResultsDiagnosticLogsFile(ResultsUploadFileInfo file)
+        {
+            Trace.Info($"Starting to upload diagnostic logs file to results service {file.Name}, {file.Path}");
+            ResultsFileUploadHandler diagnosticLogsHandler = async (file) =>
+            {
+                await _resultsServer.CreateResultsDiagnosticLogsAsync(file.PlanId, file.JobId, file.Path, CancellationToken.None);
+            };
+
+            await UploadResultsFile(file, diagnosticLogsHandler);
+        }
+
         private async Task UploadResultsStepLogFile(ResultsUploadFileInfo file)
         {
             Trace.Info($"Starting upload of step log file to results service {file.Name}, {file.Path}");

src/Runner.Common/LaunchServer.cs

@@ -1,11 +1,12 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
+using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Sdk;
+using GitHub.Services.Common;
 using GitHub.Services.Launch.Client;
-using GitHub.Services.WebApi;
 
 namespace GitHub.Runner.Common
 {
@@ -23,8 +24,21 @@ namespace GitHub.Runner.Common
 
         public void InitializeLaunchClient(Uri uri, string token)
         {
-            var httpMessageHandler = HostContext.CreateHttpClientHandler();
-            this._launchClient = new LaunchHttpClient(uri, httpMessageHandler, token, disposeHandler: true);
+            // Using default 100 timeout
+            RawClientHttpRequestSettings settings = VssUtil.GetHttpRequestSettings(null);
+
+            // Create retry handler
+            IEnumerable<DelegatingHandler> delegatingHandlers = new List<DelegatingHandler>();
+            if (settings.MaxRetryRequest > 0)
+            {
+                delegatingHandlers = new DelegatingHandler[] { new VssHttpRetryMessageHandler(settings.MaxRetryRequest) };
+            }
+
+            // Setup RawHttpMessageHandler without credentials
+            var httpMessageHandler = new RawHttpMessageHandler(new NoOpCredentials(null), settings);
+            var pipeline = HttpClientFactory.CreatePipeline(httpMessageHandler, delegatingHandlers);
+
+            this._launchClient = new LaunchHttpClient(uri, pipeline, token, disposeHandler: true);
         }
 
         public Task<ActionDownloadInfoCollection> ResolveActionsDownloadInfoAsync(Guid planId, Guid jobId, ActionReferenceList actionReferenceList,

src/Runner.Common/ResultsServer.cs

@@ -19,7 +19,7 @@ namespace GitHub.Runner.Common
     [ServiceLocator(Default = typeof(ResultServer))]
     public interface IResultsServer : IRunnerService, IAsyncDisposable
     {
-        void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token);
+        void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token, bool useSdk);
 
         Task<bool> AppendLiveConsoleFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, long? startLine, CancellationToken cancellationToken);
 
@@ -35,6 +35,8 @@ namespace GitHub.Runner.Common
 
         Task UpdateResultsWorkflowStepsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId,
             IEnumerable<TimelineRecord> records, CancellationToken cancellationToken);
+
+        Task CreateResultsDiagnosticLogsAsync(string planId, string jobId, string file, CancellationToken cancellationToken);
     }
 
     public sealed class ResultServer : RunnerService, IResultsServer
@@ -51,9 +53,9 @@ namespace GitHub.Runner.Common
         private String _liveConsoleFeedUrl;
         private string _token;
 
-        public void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token)
+        public void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token, bool useSdk)
         {
-            this._resultsClient = CreateHttpClient(uri, token);
+            this._resultsClient = CreateHttpClient(uri, token, useSdk);
 
             _token = token;
             if (!string.IsNullOrEmpty(liveConsoleFeedUrl))
@@ -63,7 +65,7 @@ namespace GitHub.Runner.Common
             }
         }
 
-        public ResultsHttpClient CreateHttpClient(Uri uri, string token)
+        public ResultsHttpClient CreateHttpClient(Uri uri, string token, bool useSdk)
         {
             // Using default 100 timeout
             RawClientHttpRequestSettings settings = VssUtil.GetHttpRequestSettings(null);
@@ -80,7 +82,7 @@ namespace GitHub.Runner.Common
 
             var pipeline = HttpClientFactory.CreatePipeline(httpMessageHandler, delegatingHandlers);
 
-            return new ResultsHttpClient(uri, pipeline, token, disposeHandler: true);
+            return new ResultsHttpClient(uri, pipeline, token, disposeHandler: true, useSdk: useSdk);
         }
 
         public Task CreateResultsStepSummaryAsync(string planId, string jobId, Guid stepId, string file,
@@ -141,6 +143,18 @@ namespace GitHub.Runner.Common
             throw new InvalidOperationException("Results client is not initialized.");
         }
 
+        public Task CreateResultsDiagnosticLogsAsync(string planId, string jobId, string file,
+            CancellationToken cancellationToken)
+        {
+            if (_resultsClient != null)
+            {
+                return _resultsClient.UploadResultsDiagnosticLogsAsync(planId, jobId, file,
+                    cancellationToken: cancellationToken);
+            }
+
+            throw new InvalidOperationException("Results client is not initialized.");
+        }
+
         public ValueTask DisposeAsync()
         {
             CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None);

src/Runner.Common/RunServer.cs

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
 using GitHub.Actions.RunService.WebApi;
 using GitHub.DistributedTask.Pipelines;
 using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
 using Sdk.RSWebApi.Contracts;
@@ -60,8 +61,11 @@ namespace GitHub.Runner.Common
         {
             CheckConnection();
             return RetryRequest<AgentJobRequestMessage>(
-                async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, cancellationToken), cancellationToken,
-                shouldRetry: ex => ex is not TaskOrchestrationJobAlreadyAcquiredException);
+                async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, VarUtil.OS, cancellationToken), cancellationToken,
+                shouldRetry: ex =>
+                    ex is not TaskOrchestrationJobNotFoundException &&          // HTTP status 404
+                    ex is not TaskOrchestrationJobAlreadyAcquiredException &&   // HTTP status 409
+                    ex is not TaskOrchestrationJobUnprocessableException);      // HTTP status 422
         }
 
         public Task CompleteJobAsync(

src/Runner.Listener/BrokerMessageListener.cs

@@ -24,7 +24,15 @@ namespace GitHub.Runner.Listener
         private TimeSpan _getNextMessageRetryInterval;
         private TaskAgentStatus runnerStatus = TaskAgentStatus.Online;
         private CancellationTokenSource _getMessagesTokenSource;
+        private VssCredentials _creds;
+        private TaskAgentSession _session;
         private IBrokerServer _brokerServer;
+        private readonly Dictionary<string, int> _sessionCreationExceptionTracker = new();
+        private bool _accessTokenRevoked = false;
+        private readonly TimeSpan _sessionCreationRetryInterval = TimeSpan.FromSeconds(30);
+        private readonly TimeSpan _sessionConflictRetryLimit = TimeSpan.FromMinutes(4);
+        private readonly TimeSpan _clockSkewRetryLimit = TimeSpan.FromMinutes(30);
+
 
         public override void Initialize(IHostContext hostContext)
         {
@@ -34,15 +42,141 @@ namespace GitHub.Runner.Listener
             _brokerServer = HostContext.GetService<IBrokerServer>();
         }
 
-        public async Task<Boolean> CreateSessionAsync(CancellationToken token)
+        public async Task<CreateSessionResult> CreateSessionAsync(CancellationToken token)
         {
-            await RefreshBrokerConnection();
-            return await Task.FromResult(true);
+            Trace.Entering();
+
+            // Settings
+            var configManager = HostContext.GetService<IConfigurationManager>();
+            _settings = configManager.LoadSettings();
+            var serverUrl = _settings.ServerUrlV2;
+            Trace.Info(_settings);
+
+            if (string.IsNullOrEmpty(_settings.ServerUrlV2))
+            {
+                throw new InvalidOperationException("ServerUrlV2 is not set");
+            }
+
+            // Create connection.
+            Trace.Info("Loading Credentials");
+            var credMgr = HostContext.GetService<ICredentialManager>();
+            _creds = credMgr.LoadCredentials();
+
+            var agent = new TaskAgentReference
+            {
+                Id = _settings.AgentId,
+                Name = _settings.AgentName,
+                Version = BuildConstants.RunnerPackage.Version,
+                OSDescription = RuntimeInformation.OSDescription,
+            };
+            var currentProcess = Process.GetCurrentProcess();
+            string sessionName = $"{Environment.MachineName ?? "RUNNER"} (PID: {currentProcess.Id})";
+            var taskAgentSession = new TaskAgentSession(sessionName, agent);
+
+            string errorMessage = string.Empty;
+            bool encounteringError = false;
+
+            while (true)
+            {
+                token.ThrowIfCancellationRequested();
+                Trace.Info($"Attempt to create session.");
+                try
+                {
+                    Trace.Info("Connecting to the Broker Server...");
+                    await _brokerServer.ConnectAsync(new Uri(serverUrl), _creds);
+                    Trace.Info("VssConnection created");
+
+                    _term.WriteLine();
+                    _term.WriteSuccessMessage("Connected to GitHub");
+                    _term.WriteLine();
+
+                    _session = await _brokerServer.CreateSessionAsync(taskAgentSession, token);
+
+                    Trace.Info($"Session created.");
+                    if (encounteringError)
+                    {
+                        _term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
+                        _sessionCreationExceptionTracker.Clear();
+                        encounteringError = false;
+                    }
+
+                    return CreateSessionResult.Success;
+                }
+                catch (OperationCanceledException) when (token.IsCancellationRequested)
+                {
+                    Trace.Info("Session creation has been cancelled.");
+                    throw;
+                }
+                catch (TaskAgentAccessTokenExpiredException)
+                {
+                    Trace.Info("Runner OAuth token has been revoked. Session creation failed.");
+                    _accessTokenRevoked = true;
+                    throw;
+                }
+                catch (Exception ex)
+                {
+                    Trace.Error("Catch exception during create session.");
+                    Trace.Error(ex);
+
+                    if (ex is VssOAuthTokenRequestException vssOAuthEx && _creds.Federated is VssOAuthCredential vssOAuthCred)
+                    {
+                        // "invalid_client" means the runner registration has been deleted from the server.
+                        if (string.Equals(vssOAuthEx.Error, "invalid_client", StringComparison.OrdinalIgnoreCase))
+                        {
+                            _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
+                            return CreateSessionResult.Failure;
+                        }
+
+                        // Check whether we get 401 because the runner registration already removed by the service.
+                        // If the runner registration get deleted, we can't exchange oauth token.
+                        Trace.Error("Test oauth app registration.");
+                        var oauthTokenProvider = new VssOAuthTokenProvider(vssOAuthCred, new Uri(serverUrl));
+                        var authError = await oauthTokenProvider.ValidateCredentialAsync(token);
+                        if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase))
+                        {
+                            _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
+                            return CreateSessionResult.Failure;
+                        }
+                    }
+
+                    if (!IsSessionCreationExceptionRetriable(ex))
+                    {
+                        _term.WriteError($"Failed to create session. {ex.Message}");
+                        if (ex is TaskAgentSessionConflictException)
+                        {
+                            return CreateSessionResult.SessionConflict;
+                        }
+                        return CreateSessionResult.Failure;
+                    }
+
+                    if (!encounteringError) //print the message only on the first error
+                    {
+                        _term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
+                        encounteringError = true;
+                    }
+
+                    Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
+                    await HostContext.Delay(_sessionCreationRetryInterval, token);
+                }
+            }
         }
 
         public async Task DeleteSessionAsync()
         {
-            await Task.CompletedTask;
+            if (_session != null && _session.SessionId != Guid.Empty)
+            {
+                if (!_accessTokenRevoked)
+                {
+                    using (var ts = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
+                    {
+                        await _brokerServer.DeleteSessionAsync(ts.Token);
+                    }
+                }
+                else
+                {
+                    Trace.Warning("Runner OAuth token has been revoked. Skip deleting session.");
+                }
+            }
         }
 
         public void OnJobStatus(object sender, JobStatusEventArgs e)
@@ -73,12 +207,13 @@ namespace GitHub.Runner.Listener
                 _getMessagesTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
                 try
                 {
-                    message = await _brokerServer.GetRunnerMessageAsync(_getMessagesTokenSource.Token,
+                    message = await _brokerServer.GetRunnerMessageAsync(_session.SessionId,
                                                                         runnerStatus,
                                                                         BuildConstants.RunnerPackage.Version,
                                                                         VarUtil.OS,
                                                                         VarUtil.OSArchitecture,
-                                                                        _settings.DisableUpdate);
+                                                                        _settings.DisableUpdate,
+                                                                        _getMessagesTokenSource.Token);
 
                     if (message == null)
                     {
@@ -143,7 +278,7 @@ namespace GitHub.Runner.Listener
                         }
 
                         // re-create VssConnection before next retry
-                        await RefreshBrokerConnection();
+                        await RefreshBrokerConnectionAsync();
 
                         Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
                         await HostContext.Delay(_getNextMessageRetryInterval, token);
@@ -173,6 +308,11 @@ namespace GitHub.Runner.Listener
             }
         }
 
+        public async Task RefreshListenerTokenAsync(CancellationToken cancellationToken)
+        {
+            await RefreshBrokerConnectionAsync();
+        }
+
         public async Task DeleteMessageAsync(TaskAgentMessage message)
         {
             await Task.CompletedTask;
@@ -196,12 +336,84 @@ namespace GitHub.Runner.Listener
             }
         }
 
-        private async Task RefreshBrokerConnection()
+        private bool IsSessionCreationExceptionRetriable(Exception ex)
+        {
+            if (ex is TaskAgentNotFoundException)
+            {
+                Trace.Info("The runner no longer exists on the server. Stopping the runner.");
+                _term.WriteError("The runner no longer exists on the server. Please reconfigure the runner.");
+                return false;
+            }
+            else if (ex is TaskAgentSessionConflictException)
+            {
+                Trace.Info("The session for this runner already exists.");
+                _term.WriteError("A session for this runner already exists.");
+                if (_sessionCreationExceptionTracker.ContainsKey(nameof(TaskAgentSessionConflictException)))
+                {
+                    _sessionCreationExceptionTracker[nameof(TaskAgentSessionConflictException)]++;
+                    if (_sessionCreationExceptionTracker[nameof(TaskAgentSessionConflictException)] * _sessionCreationRetryInterval.TotalSeconds >= _sessionConflictRetryLimit.TotalSeconds)
+                    {
+                        Trace.Info("The session conflict exception have reached retry limit.");
+                        _term.WriteError($"Stop retry on SessionConflictException after retried for {_sessionConflictRetryLimit.TotalSeconds} seconds.");
+                        return false;
+                    }
+                }
+                else
+                {
+                    _sessionCreationExceptionTracker[nameof(TaskAgentSessionConflictException)] = 1;
+                }
+
+                Trace.Info("The session conflict exception haven't reached retry limit.");
+                return true;
+            }
+            else if (ex is VssOAuthTokenRequestException && ex.Message.Contains("Current server time is"))
+            {
+                Trace.Info("Local clock might be skewed.");
+                _term.WriteError("The local machine's clock may be out of sync with the server time by more than five minutes. Please sync your clock with your domain or internet time and try again.");
+                if (_sessionCreationExceptionTracker.ContainsKey(nameof(VssOAuthTokenRequestException)))
+                {
+                    _sessionCreationExceptionTracker[nameof(VssOAuthTokenRequestException)]++;
+                    if (_sessionCreationExceptionTracker[nameof(VssOAuthTokenRequestException)] * _sessionCreationRetryInterval.TotalSeconds >= _clockSkewRetryLimit.TotalSeconds)
+                    {
+                        Trace.Info("The OAuth token request exception have reached retry limit.");
+                        _term.WriteError($"Stopped retrying OAuth token request exception after {_clockSkewRetryLimit.TotalSeconds} seconds.");
+                        return false;
+                    }
+                }
+                else
+                {
+                    _sessionCreationExceptionTracker[nameof(VssOAuthTokenRequestException)] = 1;
+                }
+
+                Trace.Info("The OAuth token request exception haven't reached retry limit.");
+                return true;
+            }
+            else if (ex is TaskAgentPoolNotFoundException ||
+                     ex is AccessDeniedException ||
+                     ex is VssUnauthorizedException)
+            {
+                Trace.Info($"Non-retriable exception: {ex.Message}");
+                return false;
+            }
+
+            else if (ex is InvalidOperationException)
+            {
+                Trace.Info($"Non-retriable exception: {ex.Message}");
+                return false;
+            }
+            else
+            {
+                Trace.Info($"Retriable exception: {ex.Message}");
+                return true;
+            }
+        }
+
+        private async Task RefreshBrokerConnectionAsync()
         {
             var configManager = HostContext.GetService<IConfigurationManager>();
             _settings = configManager.LoadSettings();
 
-            if (_settings.ServerUrlV2 == null)
+            if (string.IsNullOrEmpty(_settings.ServerUrlV2))
             {
                 throw new InvalidOperationException("ServerUrlV2 is not set");
             }

src/Runner.Listener/ErrorThrottler.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.Runner.Common;
+using GitHub.Services.Common;
+
+namespace GitHub.Runner.Listener
+{
+    [ServiceLocator(Default = typeof(ErrorThrottler))]
+    public interface IErrorThrottler : IRunnerService
+    {
+        void Reset();
+        Task IncrementAndWaitAsync(CancellationToken token);
+    }
+
+    public sealed class ErrorThrottler : RunnerService, IErrorThrottler
+    {
+        internal static readonly TimeSpan MinBackoff = TimeSpan.FromSeconds(1);
+        internal static readonly TimeSpan MaxBackoff = TimeSpan.FromMinutes(1);
+        internal static readonly TimeSpan BackoffCoefficient = TimeSpan.FromSeconds(1);
+        private int _count = 0;
+
+        public void Reset()
+        {
+            _count = 0;
+        }
+
+        public async Task IncrementAndWaitAsync(CancellationToken token)
+        {
+            if (++_count <= 1)
+            {
+                return;
+            }
+
+            TimeSpan backoff = BackoffTimerHelper.GetExponentialBackoff(
+                attempt: _count - 2, // 0-based attempt
+                minBackoff: MinBackoff,
+                maxBackoff: MaxBackoff,
+                deltaBackoff: BackoffCoefficient);
+            Trace.Warning($"Back off {backoff.TotalSeconds} seconds before next attempt. Current consecutive error count: {_count}");
+            await HostContext.Delay(backoff, token);
+        }
+    }
+}

src/Runner.Listener/JobDispatcher.cs

@@ -35,7 +35,7 @@ namespace GitHub.Runner.Listener
     // This implementation of IJobDispatcher is not thread safe.
     // It is based on the fact that the current design of the runner is a dequeue
     // and processes one message from the message queue at a time.
-    // In addition, it only executes one job every time, 
+    // In addition, it only executes one job every time,
     // and the server will not send another job while this one is still running.
     public sealed class JobDispatcher : RunnerService, IJobDispatcher
     {
@@ -546,13 +546,27 @@ namespace GitHub.Runner.Listener
                                     Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
 
                                     var jobServer = await InitializeJobServerAsync(systemConnection);
-                                    await LogWorkerProcessUnhandledException(jobServer, message, detailInfo);
-
-                                    // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
-                                    if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase))
+                                    var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
+                                    unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
+                                    switch (jobServer)
                                     {
-                                        Trace.Info($"Finish job with result 'Failed' due to IOException.");
-                                        await ForceFailJob(jobServer, message, detailInfo);
+                                        case IJobServer js:
+                                            {
+                                                await LogWorkerProcessUnhandledException(js, message, unhandledExceptionIssue);
+                                                // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
+                                                if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase))
+                                                {
+                                                    Trace.Info($"Finish job with result 'Failed' due to IOException.");
+                                                    await ForceFailJob(js, message);
+                                                }
+
+                                                break;
+                                            }
+                                        case IRunServer rs:
+                                            await ForceFailJob(rs, message, unhandledExceptionIssue);
+                                            break;
+                                        default:
+                                            throw new NotSupportedException($"JobServer type '{jobServer.GetType().Name}' is not supported.");
                                     }
                                 }
 
@@ -629,8 +643,22 @@ namespace GitHub.Runner.Listener
                                     Trace.Info("worker process has been killed.");
                                 }
                             }
+                            catch (Exception ex)
+                            {
+                                // message send failed, this might indicate worker process is already exited or stuck.
+                                Trace.Info($"Job cancel message sending for job {message.JobId} failed, kill running worker. {ex}");
+                                workerProcessCancelTokenSource.Cancel();
+                                try
+                                {
+                                    await workerProcessTask;
+                                }
+                                catch (OperationCanceledException)
+                                {
+                                    Trace.Info("worker process has been killed.");
+                                }
+                            }
 
-                            // wait worker to exit 
+                            // wait worker to exit
                             // if worker doesn't exit within timeout, then kill worker.
                             completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));
 
@@ -1117,86 +1145,65 @@ namespace GitHub.Runner.Listener
         }
 
         // log an error issue to job level timeline record
-        private async Task LogWorkerProcessUnhandledException(IRunnerService server, Pipelines.AgentJobRequestMessage message, string detailInfo)
+        private async Task LogWorkerProcessUnhandledException(IJobServer jobServer, Pipelines.AgentJobRequestMessage message, Issue issue)
         {
-            if (server is IJobServer jobServer)
+            try
             {
-                try
-                {
-                    var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None);
-                    ArgUtil.NotNull(timeline, nameof(timeline));
+                var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None);
+                ArgUtil.NotNull(timeline, nameof(timeline));
 
-                    TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job");
-                    ArgUtil.NotNull(jobRecord, nameof(jobRecord));
+                TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job");
+                ArgUtil.NotNull(jobRecord, nameof(jobRecord));
 
-                    var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
-                    unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
-                    jobRecord.ErrorCount++;
-                    jobRecord.Issues.Add(unhandledExceptionIssue);
+                jobRecord.ErrorCount++;
+                jobRecord.Issues.Add(issue);
 
-                    if (message.Variables.TryGetValue("DistributedTask.MarkJobAsFailedOnWorkerCrash", out var markJobAsFailedOnWorkerCrash) &&
-                        StringUtil.ConvertToBoolean(markJobAsFailedOnWorkerCrash?.Value))
-                    {
-                        Trace.Info("Mark the job as failed since the worker crashed");
-                        jobRecord.Result = TaskResult.Failed;
-                        // mark the job as completed so service will pickup the result
-                        jobRecord.State = TimelineRecordState.Completed;
-                    }
+                Trace.Info("Mark the job as failed since the worker crashed");
+                jobRecord.Result = TaskResult.Failed;
+                // mark the job as completed so service will pickup the result
+                jobRecord.State = TimelineRecordState.Completed;
 
-                    await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Error("Fail to report unhandled exception from Runner.Worker process");
-                    Trace.Error(ex);
-                }
+                await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None);
             }
-            else
+            catch (Exception ex)
             {
-                Trace.Info("Job server does not support handling unhandled exception yet, error message: {0}", detailInfo);
-                return;
+                Trace.Error("Fail to report unhandled exception from Runner.Worker process");
+                Trace.Error(ex);
             }
         }
 
         // raise job completed event to fail the job.
-        private async Task ForceFailJob(IRunnerService server, Pipelines.AgentJobRequestMessage message, string detailInfo)
+        private async Task ForceFailJob(IJobServer jobServer, Pipelines.AgentJobRequestMessage message)
         {
-            if (server is IJobServer jobServer)
+            try
             {
-                try
-                {
-                    var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, TaskResult.Failed);
-                    await jobServer.RaisePlanEventAsync<JobCompletedEvent>(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, CancellationToken.None);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Error("Fail to raise JobCompletedEvent back to service.");
-                    Trace.Error(ex);
-                }
+                var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, TaskResult.Failed);
+                await jobServer.RaisePlanEventAsync<JobCompletedEvent>(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, CancellationToken.None);
             }
-            else if (server is IRunServer runServer)
+            catch (Exception ex)
             {
-                try
-                {
-                    var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
-                    var unhandledAnnotation = unhandledExceptionIssue.ToAnnotation();
-                    var jobAnnotations = new List<Annotation>();
-                    if (unhandledAnnotation.HasValue)
-                    {
-                        jobAnnotations.Add(unhandledAnnotation.Value);
-                    }
+                Trace.Error("Fail to raise JobCompletedEvent back to service.");
+                Trace.Error(ex);
+            }
+        }
 
-                    await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, CancellationToken.None);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Error("Fail to raise job completion back to service.");
-                    Trace.Error(ex);
-                }
-            }
-            else
+        private async Task ForceFailJob(IRunServer runServer, Pipelines.AgentJobRequestMessage message, Issue issue)
+        {
+            try
             {
-                throw new NotSupportedException($"Server type {server.GetType().FullName} is not supported.");
+                var annotation = issue.ToAnnotation();
+                var jobAnnotations = new List<Annotation>();
+                if (annotation.HasValue)
+                {
+                    jobAnnotations.Add(annotation.Value);
+                }
+
+                await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, CancellationToken.None);
+            }
+            catch (Exception ex)
+            {
+                Trace.Error("Fail to raise job completion back to service.");
+                Trace.Error(ex);
             }
         }
 

src/Runner.Listener/MessageListener.cs

@@ -14,16 +14,26 @@ using GitHub.Runner.Listener.Configuration;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
 using GitHub.Services.OAuth;
+using GitHub.Services.WebApi;
 
 namespace GitHub.Runner.Listener
 {
+    public enum CreateSessionResult
+    {
+        Success,
+        Failure,
+        SessionConflict
+    }
+
     [ServiceLocator(Default = typeof(MessageListener))]
     public interface IMessageListener : IRunnerService
     {
-        Task<Boolean> CreateSessionAsync(CancellationToken token);
+        Task<CreateSessionResult> CreateSessionAsync(CancellationToken token);
         Task DeleteSessionAsync();
         Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token);
         Task DeleteMessageAsync(TaskAgentMessage message);
+
+        Task RefreshListenerTokenAsync(CancellationToken token);
         void OnJobStatus(object sender, JobStatusEventArgs e);
     }
 
@@ -33,6 +43,7 @@ namespace GitHub.Runner.Listener
         private RunnerSettings _settings;
         private ITerminal _term;
         private IRunnerServer _runnerServer;
+        private IBrokerServer _brokerServer;
         private TaskAgentSession _session;
         private TimeSpan _getNextMessageRetryInterval;
         private bool _accessTokenRevoked = false;
@@ -42,6 +53,9 @@ namespace GitHub.Runner.Listener
         private readonly Dictionary<string, int> _sessionCreationExceptionTracker = new();
         private TaskAgentStatus runnerStatus = TaskAgentStatus.Online;
         private CancellationTokenSource _getMessagesTokenSource;
+        private VssCredentials _creds;
+
+        private bool _isBrokerSession = false;
 
         public override void Initialize(IHostContext hostContext)
         {
@@ -49,9 +63,10 @@ namespace GitHub.Runner.Listener
 
             _term = HostContext.GetService<ITerminal>();
             _runnerServer = HostContext.GetService<IRunnerServer>();
+            _brokerServer = hostContext.GetService<IBrokerServer>();
         }
 
-        public async Task<Boolean> CreateSessionAsync(CancellationToken token)
+        public async Task<CreateSessionResult> CreateSessionAsync(CancellationToken token)
         {
             Trace.Entering();
 
@@ -64,7 +79,7 @@ namespace GitHub.Runner.Listener
             // Create connection.
             Trace.Info("Loading Credentials");
             var credMgr = HostContext.GetService<ICredentialManager>();
-            VssCredentials creds = credMgr.LoadCredentials();
+            _creds = credMgr.LoadCredentials();
 
             var agent = new TaskAgentReference
             {
@@ -73,7 +88,8 @@ namespace GitHub.Runner.Listener
                 Version = BuildConstants.RunnerPackage.Version,
                 OSDescription = RuntimeInformation.OSDescription,
             };
-            string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
+            var currentProcess = Process.GetCurrentProcess();
+            string sessionName = $"{Environment.MachineName ?? "RUNNER"} (PID: {currentProcess.Id})";
             var taskAgentSession = new TaskAgentSession(sessionName, agent);
 
             string errorMessage = string.Empty;
@@ -86,7 +102,7 @@ namespace GitHub.Runner.Listener
                 try
                 {
                     Trace.Info("Connecting to the Runner Server...");
-                    await _runnerServer.ConnectAsync(new Uri(serverUrl), creds);
+                    await _runnerServer.ConnectAsync(new Uri(serverUrl), _creds);
                     Trace.Info("VssConnection created");
 
                     _term.WriteLine();
@@ -98,6 +114,15 @@ namespace GitHub.Runner.Listener
                                                         taskAgentSession,
                                                         token);
 
+                    if (_session.BrokerMigrationMessage != null)
+                    {
+                        Trace.Info("Runner session is in migration mode: Creating Broker session with BrokerBaseUrl: {0}", _session.BrokerMigrationMessage.BrokerBaseUrl);
+
+                        await _brokerServer.UpdateConnectionIfNeeded(_session.BrokerMigrationMessage.BrokerBaseUrl, _creds);
+                        _session = await _brokerServer.CreateSessionAsync(taskAgentSession, token);
+                        _isBrokerSession = true;
+                    }
+
                     Trace.Info($"Session created.");
                     if (encounteringError)
                     {
@@ -106,7 +131,7 @@ namespace GitHub.Runner.Listener
                         encounteringError = false;
                     }
 
-                    return true;
+                    return CreateSessionResult.Success;
                 }
                 catch (OperationCanceledException) when (token.IsCancellationRequested)
                 {
@@ -124,13 +149,13 @@ namespace GitHub.Runner.Listener
                     Trace.Error("Catch exception during create session.");
                     Trace.Error(ex);
 
-                    if (ex is VssOAuthTokenRequestException vssOAuthEx && creds.Federated is VssOAuthCredential vssOAuthCred)
+                    if (ex is VssOAuthTokenRequestException vssOAuthEx && _creds.Federated is VssOAuthCredential vssOAuthCred)
                     {
                         // "invalid_client" means the runner registration has been deleted from the server.
                         if (string.Equals(vssOAuthEx.Error, "invalid_client", StringComparison.OrdinalIgnoreCase))
                         {
                             _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
-                            return false;
+                            return CreateSessionResult.Failure;
                         }
 
                         // Check whether we get 401 because the runner registration already removed by the service.
@@ -141,14 +166,18 @@ namespace GitHub.Runner.Listener
                         if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase))
                         {
                             _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
-                            return false;
+                            return CreateSessionResult.Failure;
                         }
                     }
 
                     if (!IsSessionCreationExceptionRetriable(ex))
                     {
                         _term.WriteError($"Failed to create session. {ex.Message}");
-                        return false;
+                        if (ex is TaskAgentSessionConflictException)
+                        {
+                            return CreateSessionResult.SessionConflict;
+                        }
+                        return CreateSessionResult.Failure;
                     }
 
                     if (!encounteringError) //print the message only on the first error
@@ -172,6 +201,11 @@ namespace GitHub.Runner.Listener
                     using (var ts = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
                     {
                         await _runnerServer.DeleteAgentSessionAsync(_settings.PoolId, _session.SessionId, ts.Token);
+
+                        if (_isBrokerSession)
+                        {
+                            await _brokerServer.DeleteSessionAsync(ts.Token);
+                        }
                     }
                 }
                 else
@@ -193,6 +227,7 @@ namespace GitHub.Runner.Listener
             {
                 Trace.Info("_getMessagesTokenSource is already disposed.");
             }
+
         }
 
         public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
@@ -202,6 +237,7 @@ namespace GitHub.Runner.Listener
             ArgUtil.NotNull(_settings, nameof(_settings));
             bool encounteringError = false;
             int continuousError = 0;
+            int continuousEmptyMessage = 0;
             string errorMessage = string.Empty;
             Stopwatch heartbeat = new();
             heartbeat.Restart();
@@ -225,6 +261,23 @@ namespace GitHub.Runner.Listener
                     // Decrypt the message body if the session is using encryption
                     message = DecryptMessage(message);
 
+
+                    if (message != null && message.MessageType == BrokerMigrationMessage.MessageType)
+                    {
+                        Trace.Info("BrokerMigration message received. Polling Broker for messages...");
+
+                        var migrationMessage = JsonUtility.FromString<BrokerMigrationMessage>(message.Body);
+
+                        await _brokerServer.UpdateConnectionIfNeeded(migrationMessage.BrokerBaseUrl, _creds);
+                        message = await _brokerServer.GetRunnerMessageAsync(_session.SessionId,
+                                                                        runnerStatus,
+                                                                        BuildConstants.RunnerPackage.Version,
+                                                                        VarUtil.OS,
+                                                                        VarUtil.OSArchitecture,
+                                                                        _settings.DisableUpdate,
+                                                                        token);
+                    }
+
                     if (message != null)
                     {
                         _lastMessageId = message.MessageId;
@@ -263,7 +316,7 @@ namespace GitHub.Runner.Listener
                     Trace.Error(ex);
 
                     // don't retry if SkipSessionRecover = true, DT service will delete agent session to stop agent from taking more jobs.
-                    if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && await CreateSessionAsync(token))
+                    if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && (await CreateSessionAsync(token) == CreateSessionResult.Success))
                     {
                         Trace.Info($"{nameof(TaskAgentSessionExpiredException)} received, recovered by recreate session.");
                     }
@@ -308,16 +361,27 @@ namespace GitHub.Runner.Listener
 
                 if (message == null)
                 {
+                    continuousEmptyMessage++;
                     if (heartbeat.Elapsed > TimeSpan.FromMinutes(30))
                     {
                         Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes.");
                         heartbeat.Restart();
+                        continuousEmptyMessage = 0;
                     }
                     else
                     {
                         Trace.Verbose($"No message retrieved from session '{_session.SessionId}'.");
                     }
 
+                    if (continuousEmptyMessage > 50)
+                    {
+                        // retried more than 50 times in less than 30mins and still getting empty message
+                        // something is not right on the service side, backoff for 15-30s before retry
+                        _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval);
+                        Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
+                        await HostContext.Delay(_getNextMessageRetryInterval, token);
+                    }
+
                     continue;
                 }
 
@@ -340,6 +404,12 @@ namespace GitHub.Runner.Listener
             }
         }
 
+        public async Task RefreshListenerTokenAsync(CancellationToken cancellationToken)
+        {
+            await _runnerServer.RefreshConnectionAsync(RunnerConnectionType.MessageQueue, TimeSpan.FromSeconds(60));
+            await _brokerServer.ForceRefreshConnection(_creds);
+        }
+
         private TaskAgentMessage DecryptMessage(TaskAgentMessage message)
         {
             if (_session.EncryptionKey == null ||

src/Runner.Listener/Runner.Listener.csproj

@@ -25,12 +25,6 @@
     <PackageReference Include="System.ServiceProcess.ServiceController" Version="4.4.0" />
   </ItemGroup>
 
-  <ItemGroup>
-    <EmbeddedResource Include="..\Misc\runnercoreassets">
-      <LogicalName>GitHub.Runner.Listener.runnercoreassets</LogicalName>
-    </EmbeddedResource>
-  </ItemGroup>
-
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
     <DebugType>portable</DebugType>
   </PropertyGroup>

src/Runner.Listener/Runner.cs

@@ -32,10 +32,25 @@ namespace GitHub.Runner.Listener
         private bool _inConfigStage;
         private ManualResetEvent _completedCommand = new(false);
 
+        // <summary>
+        // Helps avoid excessive calls to Run Service when encountering non-retriable errors from /acquirejob.
+        // Normally we rely on the HTTP clients to back off between retry attempts. However, acquiring a job
+        // involves calls to both Run Serivce and Broker. And Run Service and Broker communicate with each other
+        // in an async fashion.
+        //
+        // When Run Service encounters a non-retriable error, it sends an async message to Broker. The runner will,
+        // however, immediately call Broker to get the next message. If the async event from Run Service to Broker
+        // has not yet been processed, the next message from Broker may be the same job message.
+        //
+        // The error throttler helps us back off when encountering successive, non-retriable errors from /acquirejob.
+        // </summary>
+        private IErrorThrottler _acquireJobThrottler;
+
         public override void Initialize(IHostContext hostContext)
         {
             base.Initialize(hostContext);
             _term = HostContext.GetService<ITerminal>();
+            _acquireJobThrottler = HostContext.CreateService<IErrorThrottler>();
         }
 
         public async Task<int> ExecuteCommand(CommandSettings command)
@@ -222,6 +237,10 @@ namespace GitHub.Runner.Listener
                             File.SetAttributes(configFile, File.GetAttributes(configFile) | FileAttributes.Hidden);
                             Trace.Info($"Saved {configContent.Length} bytes to '{configFile}'.");
                         }
+
+                        // make sure we have the right user agent data added from the jitconfig
+                        HostContext.LoadDefaultUserAgents();
+                        VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy);
                     }
                     catch (Exception ex)
                     {
@@ -359,7 +378,12 @@ namespace GitHub.Runner.Listener
             {
                 Trace.Info(nameof(RunAsync));
                 _listener = GetMesageListener(settings);
-                if (!await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken))
+                CreateSessionResult createSessionResult = await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken);
+                if (createSessionResult == CreateSessionResult.SessionConflict)
+                {
+                    return Constants.Runner.ReturnCode.SessionConflict;
+                }
+                else if (createSessionResult == CreateSessionResult.Failure)
                 {
                     return Constants.Runner.ReturnCode.TerminatedError;
                 }
@@ -558,13 +582,21 @@ namespace GitHub.Runner.Listener
                                         await runServer.ConnectAsync(new Uri(messageRef.RunServiceUrl), creds);
                                         try
                                         {
-                                            jobRequestMessage =
-                                            await runServer.GetJobMessageAsync(messageRef.RunnerRequestId,
-                                            messageQueueLoopTokenSource.Token);
+                                            jobRequestMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageQueueLoopTokenSource.Token);
+                                            _acquireJobThrottler.Reset();
                                         }
-                                        catch (TaskOrchestrationJobAlreadyAcquiredException)
+                                        catch (Exception ex) when (
+                                            ex is TaskOrchestrationJobNotFoundException ||          // HTTP status 404
+                                            ex is TaskOrchestrationJobAlreadyAcquiredException ||   // HTTP status 409
+                                            ex is TaskOrchestrationJobUnprocessableException)       // HTTP status 422
                                         {
-                                            Trace.Info("Job is already acquired, skip this message.");
+                                            Trace.Info($"Skipping message Job. {ex.Message}");
+                                            await _acquireJobThrottler.IncrementAndWaitAsync(messageQueueLoopTokenSource.Token);
+                                            continue;
+                                        }
+                                        catch (Exception ex)
+                                        {
+                                            Trace.Error($"Caught exception from acquiring job message: {ex}");
                                             continue;
                                         }
                                     }
@@ -596,6 +628,11 @@ namespace GitHub.Runner.Listener
                                 Trace.Info($"Service requests the hosted runner to shutdown. Reason: '{HostedRunnerShutdownMessage.Reason}'.");
                                 return Constants.Runner.ReturnCode.Success;
                             }
+                            else if (string.Equals(message.MessageType, TaskAgentMessageTypes.ForceTokenRefresh))
+                            {
+                                Trace.Info("Received ForceTokenRefreshMessage");
+                                await _listener.RefreshListenerTokenAsync(messageQueueLoopTokenSource.Token);
+                            }
                             else
                             {
                                 Trace.Error($"Received message {message.MessageId} with unsupported message type {message.MessageType}.");
@@ -634,6 +671,7 @@ namespace GitHub.Runner.Listener
                     {
                         try
                         {
+                            Trace.Info("Deleting Runner Session...");
                             await _listener.DeleteSessionAsync();
                         }
                         catch (Exception ex) when (runOnce)

src/Runner.Listener/SelfUpdater.cs

@@ -6,13 +6,11 @@ using System.IO;
 using System.IO.Compression;
 using System.Linq;
 using System.Net.Http;
-using System.Reflection;
 using System.Security.Cryptography;
 using System.Threading;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
-using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
 using GitHub.Services.WebApi;
@@ -30,20 +28,14 @@ namespace GitHub.Runner.Listener
     {
         private static string _packageType = "agent";
         private static string _platform = BuildConstants.RunnerPackage.PackageName;
-        private static string _dotnetRuntime = "dotnetRuntime";
-        private static string _externals = "externals";
-        private readonly Dictionary<string, string> _contentHashes = new();
 
         private PackageMetadata _targetPackage;
         private ITerminal _terminal;
         private IRunnerServer _runnerServer;
         private int _poolId;
         private ulong _agentId;
+        private const int _numberOfOldVersionsToKeep = 1;
         private readonly ConcurrentQueue<string> _updateTrace = new();
-        private Task _cloneAndCalculateContentHashTask;
-        private string _dotnetRuntimeCloneDirectory;
-        private string _externalsCloneDirectory;
-
         public bool Busy { get; private set; }
 
         public override void Initialize(IHostContext hostContext)
@@ -56,8 +48,6 @@ namespace GitHub.Runner.Listener
             var settings = configStore.GetSettings();
             _poolId = settings.PoolId;
             _agentId = settings.AgentId;
-            _dotnetRuntimeCloneDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), "__dotnet_runtime__");
-            _externalsCloneDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), "__externals__");
         }
 
         public async Task<bool> SelfUpdate(AgentRefreshMessage updateMessage, IJobDispatcher jobDispatcher, bool restartInteractiveRunner, CancellationToken token)
@@ -67,13 +57,6 @@ namespace GitHub.Runner.Listener
             {
                 var totalUpdateTime = Stopwatch.StartNew();
 
-                // Copy dotnet runtime and externals of current runner to a temp folder
-                // So we can re-use them with trimmed runner package, if possible.
-                // This process is best effort, if we can't use trimmed runner package, 
-                // we will just go with the full package.
-                var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
-                _cloneAndCalculateContentHashTask = CloneAndCalculateAssetsHash(_dotnetRuntimeCloneDirectory, _externalsCloneDirectory, linkedTokenSource.Token);
-
                 if (!await UpdateNeeded(updateMessage.TargetVersion, token))
                 {
                     Trace.Info($"Can't find available update package.");
@@ -87,24 +70,6 @@ namespace GitHub.Runner.Listener
                 await UpdateRunnerUpdateStateAsync("Runner update in progress, do not shutdown runner.");
                 await UpdateRunnerUpdateStateAsync($"Downloading {_targetPackage.Version} runner");
 
-                if (_targetPackage.TrimmedPackages?.Count > 0)
-                {
-                    // wait for cloning assets task to finish only if we have trimmed packages
-                    await _cloneAndCalculateContentHashTask;
-                }
-                else
-                {
-                    linkedTokenSource.Cancel();
-                    try
-                    {
-                        await _cloneAndCalculateContentHashTask;
-                    }
-                    catch (Exception ex)
-                    {
-                        Trace.Info($"Ingore errors after cancelling cloning assets task: {ex}");
-                    }
-                }
-
                 await DownloadLatestRunner(token, updateMessage.TargetVersion);
                 Trace.Info($"Download latest runner and unzip into runner root.");
 
@@ -218,54 +183,8 @@ namespace GitHub.Runner.Listener
             string archiveFile = null;
             var packageDownloadUrl = _targetPackage.DownloadUrl;
             var packageHashValue = _targetPackage.HashValue;
-            var runtimeTrimmed = false;
-            var externalsTrimmed = false;
-            var fallbackToFullPackage = false;
-
-            // Only try trimmed package if sever sends them and we have calculated hash value of the current runtime/externals.
-            if (_contentHashes.Count == 2 &&
-                _contentHashes.ContainsKey(_dotnetRuntime) &&
-                _contentHashes.ContainsKey(_externals) &&
-                _targetPackage.TrimmedPackages?.Count > 0)
-            {
-                Trace.Info($"Current runner content hash: {StringUtil.ConvertToJson(_contentHashes)}");
-                Trace.Info($"Trimmed packages info from service: {StringUtil.ConvertToJson(_targetPackage.TrimmedPackages)}");
-                // Try to see whether we can use any size trimmed down package to speed up runner updates.
-                foreach (var trimmedPackage in _targetPackage.TrimmedPackages)
-                {
-                    if (trimmedPackage.TrimmedContents.Count == 2 &&
-                        trimmedPackage.TrimmedContents.TryGetValue(_dotnetRuntime, out var trimmedRuntimeHash) &&
-                        trimmedRuntimeHash == _contentHashes[_dotnetRuntime] &&
-                        trimmedPackage.TrimmedContents.TryGetValue(_externals, out var trimmedExternalsHash) &&
-                        trimmedExternalsHash == _contentHashes[_externals])
-                    {
-                        Trace.Info($"Use trimmed (runtime+externals) package '{trimmedPackage.DownloadUrl}' to update runner.");
-                        packageDownloadUrl = trimmedPackage.DownloadUrl;
-                        packageHashValue = trimmedPackage.HashValue;
-                        runtimeTrimmed = true;
-                        externalsTrimmed = true;
-                        break;
-                    }
-                    else if (trimmedPackage.TrimmedContents.Count == 1 &&
-                             trimmedPackage.TrimmedContents.TryGetValue(_externals, out trimmedExternalsHash) &&
-                             trimmedExternalsHash == _contentHashes[_externals])
-                    {
-                        Trace.Info($"Use trimmed (externals) package '{trimmedPackage.DownloadUrl}' to update runner.");
-                        packageDownloadUrl = trimmedPackage.DownloadUrl;
-                        packageHashValue = trimmedPackage.HashValue;
-                        externalsTrimmed = true;
-                        break;
-                    }
-                    else
-                    {
-                        Trace.Info($"Can't use trimmed package from '{trimmedPackage.DownloadUrl}' since the current runner does not carry those trimmed content (Hash mismatch).");
-                    }
-                }
-            }
 
             _updateTrace.Enqueue($"DownloadUrl: {packageDownloadUrl}");
-            _updateTrace.Enqueue($"RuntimeTrimmed: {runtimeTrimmed}");
-            _updateTrace.Enqueue($"ExternalsTrimmed: {externalsTrimmed}");
 
             try
             {
@@ -323,12 +242,6 @@ namespace GitHub.Runner.Listener
 
                 await ExtractRunnerPackage(archiveFile, latestRunnerDirectory, token);
             }
-            catch (Exception ex) when (runtimeTrimmed || externalsTrimmed)
-            {
-                // if anything failed when we use trimmed package (download/validatehase/extract), try again with the full runner package.
-                Trace.Error($"Fail to download latest runner using trimmed package: {ex}");
-                fallbackToFullPackage = true;
-            }
             finally
             {
                 try
@@ -347,74 +260,6 @@ namespace GitHub.Runner.Listener
                 }
             }
 
-            var trimmedPackageRestoreTasks = new List<Task<bool>>();
-            if (!fallbackToFullPackage)
-            {
-                // Skip restoring externals and runtime if we are going to fullback to the full package.
-                if (externalsTrimmed)
-                {
-                    trimmedPackageRestoreTasks.Add(RestoreTrimmedExternals(latestRunnerDirectory, token));
-                }
-                if (runtimeTrimmed)
-                {
-                    trimmedPackageRestoreTasks.Add(RestoreTrimmedDotnetRuntime(latestRunnerDirectory, token));
-                }
-            }
-
-            if (trimmedPackageRestoreTasks.Count > 0)
-            {
-                var restoreResults = await Task.WhenAll(trimmedPackageRestoreTasks);
-                if (restoreResults.Any(x => x == false))
-                {
-                    // if any of the restore failed, fallback to full package.
-                    fallbackToFullPackage = true;
-                }
-            }
-
-            if (fallbackToFullPackage)
-            {
-                Trace.Error("Something wrong with the trimmed runner package, failback to use the full package for runner updates.");
-                _updateTrace.Enqueue($"FallbackToFullPackage: {fallbackToFullPackage}");
-
-                IOUtil.DeleteDirectory(latestRunnerDirectory, token);
-                Directory.CreateDirectory(latestRunnerDirectory);
-
-                packageDownloadUrl = _targetPackage.DownloadUrl;
-                packageHashValue = _targetPackage.HashValue;
-                _updateTrace.Enqueue($"DownloadUrl: {packageDownloadUrl}");
-
-                try
-                {
-                    archiveFile = await DownLoadRunner(latestRunnerDirectory, packageDownloadUrl, packageHashValue, token);
-
-                    if (string.IsNullOrEmpty(archiveFile))
-                    {
-                        throw new TaskCanceledException($"Runner package '{packageDownloadUrl}' failed after {Constants.RunnerDownloadRetryMaxAttempts} download attempts");
-                    }
-
-                    await ValidateRunnerHash(archiveFile, packageHashValue);
-
-                    await ExtractRunnerPackage(archiveFile, latestRunnerDirectory, token);
-                }
-                finally
-                {
-                    try
-                    {
-                        // delete .zip file
-                        if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
-                        {
-                            Trace.Verbose("Deleting latest runner package zip: {0}", archiveFile);
-                            IOUtil.DeleteFile(archiveFile);
-                        }
-                    }
-                    catch (Exception ex)
-                    {
-                        //it is not critical if we fail to delete the .zip file
-                        Trace.Warning("Failed to delete runner package zip '{0}'. Exception: {1}", archiveFile, ex);
-                    }
-                }
-            }
-
             await CopyLatestRunnerToRoot(latestRunnerDirectory, token);
         }
 
@@ -665,9 +510,9 @@ namespace GitHub.Runner.Listener
 
             // delete old bin.2.99.0 folder, only leave the current version and the latest download version
             var allBinDirs = Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "bin.*");
-            if (allBinDirs.Length > 2)
+            if (allBinDirs.Length > _numberOfOldVersionsToKeep)
             {
-                // there are more than 2 bin.version folder.
+                // there are more than one bin.version folder.
                 // delete older bin.version folders.
                 foreach (var oldBinDir in allBinDirs)
                 {
@@ -694,9 +539,9 @@ namespace GitHub.Runner.Listener
 
             // delete old externals.2.99.0 folder, only leave the current version and the latest download version
             var allExternalsDirs = Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "externals.*");
-            if (allExternalsDirs.Length > 2)
+            if (allExternalsDirs.Length > _numberOfOldVersionsToKeep)
             {
-                // there are more than 2 externals.version folder.
+                // there are more than one externals.version folder.
                 // delete older externals.version folders.
                 foreach (var oldExternalDir in allExternalsDirs)
                 {
@@ -795,330 +640,5 @@ namespace GitHub.Runner.Listener
                 Trace.Info($"Catch exception during report update state, ignore this error and continue auto-update.");
             }
         }
-
-        private async Task<bool> RestoreTrimmedExternals(string downloadDirectory, CancellationToken token)
-        {
-            // Copy the current runner's externals if we are using a externals trimmed package
-            // Execute the node.js to make sure the copied externals is working.
-            var stopWatch = Stopwatch.StartNew();
-            try
-            {
-                Trace.Info($"Copy {_externalsCloneDirectory} to {Path.Combine(downloadDirectory, Constants.Path.ExternalsDirectory)}.");
-                IOUtil.CopyDirectory(_externalsCloneDirectory, Path.Combine(downloadDirectory, Constants.Path.ExternalsDirectory), token);
-
-                // try run node.js to see if current node.js works fine after copy over to new location.
-                var nodeVersions = NodeUtil.BuiltInNodeVersions;
-                foreach (var nodeVersion in nodeVersions)
-                {
-                    var newNodeBinary = Path.Combine(downloadDirectory, Constants.Path.ExternalsDirectory, nodeVersion, "bin", $"node{IOUtil.ExeExtension}");
-                    if (File.Exists(newNodeBinary))
-                    {
-                        using (var p = HostContext.CreateService<IProcessInvoker>())
-                        {
-                            var outputs = "";
-                            p.ErrorDataReceived += (_, data) =>
-                            {
-                                if (!string.IsNullOrEmpty(data.Data))
-                                {
-                                    Trace.Error(data.Data);
-                                }
-                            };
-                            p.OutputDataReceived += (_, data) =>
-                            {
-                                if (!string.IsNullOrEmpty(data.Data))
-                                {
-                                    Trace.Info(data.Data);
-                                    outputs = data.Data;
-                                }
-                            };
-                            var exitCode = await p.ExecuteAsync(HostContext.GetDirectory(WellKnownDirectory.Root), newNodeBinary, $"-e \"console.log('{nameof(RestoreTrimmedExternals)}')\"", null, token);
-                            if (exitCode != 0)
-                            {
-                                Trace.Error($"{newNodeBinary} -e \"console.log()\" failed with exit code {exitCode}");
-                                return false;
-                            }
-
-                            if (!string.Equals(outputs, nameof(RestoreTrimmedExternals), StringComparison.OrdinalIgnoreCase))
-                            {
-                                Trace.Error($"{newNodeBinary} -e \"console.log()\" did not output expected content.");
-                                return false;
-                            }
-                        }
-                    }
-                }
-
-                return true;
-            }
-            catch (Exception ex)
-            {
-                Trace.Error($"Fail to restore externals for trimmed package: {ex}");
-                return false;
-            }
-            finally
-            {
-                stopWatch.Stop();
-                _updateTrace.Enqueue($"{nameof(RestoreTrimmedExternals)}Time: {stopWatch.ElapsedMilliseconds}ms");
-            }
-        }
-
-        private async Task<bool> RestoreTrimmedDotnetRuntime(string downloadDirectory, CancellationToken token)
-        {
-            // Copy the current runner's dotnet runtime if we are using a dotnet runtime trimmed package
-            // Execute the runner.listener to make sure the copied runtime is working.
-            var stopWatch = Stopwatch.StartNew();
-            try
-            {
-                Trace.Info($"Copy {_dotnetRuntimeCloneDirectory} to {Path.Combine(downloadDirectory, Constants.Path.BinDirectory)}.");
-                IOUtil.CopyDirectory(_dotnetRuntimeCloneDirectory, Path.Combine(downloadDirectory, Constants.Path.BinDirectory), token);
-
-                // try run the runner executable to see if current dotnet runtime + future runner binary works fine.
-                var newRunnerBinary = Path.Combine(downloadDirectory, Constants.Path.BinDirectory, "Runner.Listener");
-                using (var p = HostContext.CreateService<IProcessInvoker>())
-                {
-                    p.ErrorDataReceived += (_, data) =>
-                    {
-                        if (!string.IsNullOrEmpty(data.Data))
-                        {
-                            Trace.Error(data.Data);
-                        }
-                    };
-                    p.OutputDataReceived += (_, data) =>
-                    {
-                        if (!string.IsNullOrEmpty(data.Data))
-                        {
-                            Trace.Info(data.Data);
-                        }
-                    };
-                    var exitCode = await p.ExecuteAsync(HostContext.GetDirectory(WellKnownDirectory.Root), newRunnerBinary, "--version", null, token);
-                    if (exitCode != 0)
-                    {
-                        Trace.Error($"{newRunnerBinary} --version failed with exit code {exitCode}");
-                        return false;
-                    }
-                    else
-                    {
-                        return true;
-                    }
-                }
-            }
-            catch (Exception ex)
-            {
-                Trace.Error($"Fail to restore dotnet runtime for trimmed package: {ex}");
-                return false;
-            }
-            finally
-            {
-                stopWatch.Stop();
-                _updateTrace.Enqueue($"{nameof(RestoreTrimmedDotnetRuntime)}Time: {stopWatch.ElapsedMilliseconds}ms");
-            }
-        }
-
-        private async Task CloneAndCalculateAssetsHash(string dotnetRuntimeCloneDirectory, string externalsCloneDirectory, CancellationToken token)
-        {
-            var runtimeCloneTask = CloneDotnetRuntime(dotnetRuntimeCloneDirectory, token);
-            var externalsCloneTask = CloneExternals(externalsCloneDirectory, token);
-
-            var waitingTasks = new Dictionary<string, Task>()
-            {
-                {nameof(CloneDotnetRuntime), runtimeCloneTask},
-                {nameof(CloneExternals),externalsCloneTask}
-            };
-
-            while (waitingTasks.Count > 0)
-            {
-                Trace.Info($"Waiting for {waitingTasks.Count} tasks to complete.");
-                var complatedTask = await Task.WhenAny(waitingTasks.Values);
-                if (waitingTasks.ContainsKey(nameof(CloneExternals)) &&
-                    complatedTask == waitingTasks[nameof(CloneExternals)])
-                {
-                    Trace.Info($"Externals clone finished.");
-                    waitingTasks.Remove(nameof(CloneExternals));
-                    try
-                    {
-                        if (await externalsCloneTask && !token.IsCancellationRequested)
-                        {
-                            var externalsHash = await HashFiles(externalsCloneDirectory, token);
-                            Trace.Info($"Externals content hash: {externalsHash}");
-                            _contentHashes[_externals] = externalsHash;
-                            _updateTrace.Enqueue($"ExternalsHash: {_contentHashes[_externals]}");
-                        }
-                        else
-                        {
-                            Trace.Error($"Skip compute hash since clone externals failed/cancelled.");
-                        }
-                    }
-                    catch (Exception ex)
-                    {
-                        Trace.Error($"Fail to hash externals content: {ex}");
-                    }
-                }
-                else if (waitingTasks.ContainsKey(nameof(CloneDotnetRuntime)) &&
-                         complatedTask == waitingTasks[nameof(CloneDotnetRuntime)])
-                {
-                    Trace.Info($"Dotnet runtime clone finished.");
-                    waitingTasks.Remove(nameof(CloneDotnetRuntime));
-                    try
-                    {
-                        if (await runtimeCloneTask && !token.IsCancellationRequested)
-                        {
-                            var runtimeHash = await HashFiles(dotnetRuntimeCloneDirectory, token);
-                            Trace.Info($"Runtime content hash: {runtimeHash}");
-                            _contentHashes[_dotnetRuntime] = runtimeHash;
-                            _updateTrace.Enqueue($"DotnetRuntimeHash: {_contentHashes[_dotnetRuntime]}");
-                        }
-                        else
-                        {
-                            Trace.Error($"Skip compute hash since clone dotnet runtime failed/cancelled.");
-                        }
-                    }
-                    catch (Exception ex)
-                    {
-                        Trace.Error($"Fail to hash runtime content: {ex}");
-                    }
-                }
-
-                Trace.Info($"Still waiting for {waitingTasks.Count} tasks to complete.");
-            }
-        }
-
-        private async Task<bool> CloneDotnetRuntime(string runtimeDir, CancellationToken token)
-        {
-            var stopWatch = Stopwatch.StartNew();
-            try
-            {
-                Trace.Info($"Cloning dotnet runtime to {runtimeDir}");
-                IOUtil.DeleteDirectory(runtimeDir, CancellationToken.None);
-                Directory.CreateDirectory(runtimeDir);
-
-                var assembly = Assembly.GetExecutingAssembly();
-                var assetsContent = default(string);
-                using (var stream = assembly.GetManifestResourceStream("GitHub.Runner.Listener.runnercoreassets"))
-                using (var streamReader = new StreamReader(stream))
-                {
-                    assetsContent = await streamReader.ReadToEndAsync();
-                }
-
-                if (!string.IsNullOrEmpty(assetsContent))
-                {
-                    var runnerCoreAssets = assetsContent.Split(new[] { "\n", "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
-                    if (runnerCoreAssets.Length > 0)
-                    {
-                        var binDir = HostContext.GetDirectory(WellKnownDirectory.Bin);
-                        IOUtil.CopyDirectory(binDir, runtimeDir, token);
-
-                        var clonedFile = 0;
-                        foreach (var file in Directory.EnumerateFiles(runtimeDir, "*", SearchOption.AllDirectories))
-                        {
-                            token.ThrowIfCancellationRequested();
-                            if (runnerCoreAssets.Any(x => file.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).EndsWith(x.Trim())))
-                            {
-                                Trace.Verbose($"{file} is part of the runner core, delete from cloned runtime directory.");
-                                IOUtil.DeleteFile(file);
-                            }
-                            else
-                            {
-                                clonedFile++;
-                            }
-                        }
-
-                        Trace.Info($"Successfully cloned dotnet runtime to {runtimeDir}. Total files: {clonedFile}");
-                        return true;
-                    }
-                }
-            }
-            catch (Exception ex)
-            {
-                Trace.Error($"Fail to clone dotnet runtime to {runtimeDir}");
-                Trace.Error(ex);
-            }
-            finally
-            {
-                stopWatch.Stop();
-                _updateTrace.Enqueue($"{nameof(CloneDotnetRuntime)}Time: {stopWatch.ElapsedMilliseconds}ms");
-            }
-
-            return false;
-        }
-
-        private Task<bool> CloneExternals(string externalsDir, CancellationToken token)
-        {
-            var stopWatch = Stopwatch.StartNew();
-            try
-            {
-                Trace.Info($"Cloning externals to {externalsDir}");
-                IOUtil.DeleteDirectory(externalsDir, CancellationToken.None);
-                Directory.CreateDirectory(externalsDir);
-                IOUtil.CopyDirectory(HostContext.GetDirectory(WellKnownDirectory.Externals), externalsDir, token);
-                Trace.Info($"Successfully cloned externals to {externalsDir}.");
-                return Task.FromResult(true);
-            }
-            catch (Exception ex)
-            {
-                Trace.Error($"Fail to clone externals to {externalsDir}");
-                Trace.Error(ex);
-            }
-            finally
-            {
-                stopWatch.Stop();
-                _updateTrace.Enqueue($"{nameof(CloneExternals)}Time: {stopWatch.ElapsedMilliseconds}ms");
-            }
-
-            return Task.FromResult(false);
-        }
-
-        private async Task<string> HashFiles(string fileFolder, CancellationToken token)
-        {
-            Trace.Info($"Calculating hash for {fileFolder}");
-
-            var stopWatch = Stopwatch.StartNew();
-            string binDir = HostContext.GetDirectory(WellKnownDirectory.Bin);
-            string node = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), NodeUtil.GetInternalNodeVersion(), "bin", $"node{IOUtil.ExeExtension}");
-            string hashFilesScript = Path.Combine(binDir, "hashFiles");
-            var hashResult = string.Empty;
-
-            using (var processInvoker = HostContext.CreateService<IProcessInvoker>())
-            {
-                processInvoker.ErrorDataReceived += (_, data) =>
-                {
-                    if (!string.IsNullOrEmpty(data.Data) && data.Data.StartsWith("__OUTPUT__") && data.Data.EndsWith("__OUTPUT__"))
-                    {
-                        hashResult = data.Data.Substring(10, data.Data.Length - 20);
-                        Trace.Info($"Hash result: '{hashResult}'");
-                    }
-                    else
-                    {
-                        Trace.Info(data.Data);
-                    }
-                };
-
-                processInvoker.OutputDataReceived += (_, data) =>
-                {
-                    Trace.Verbose(data.Data);
-                };
-
-                var env = new Dictionary<string, string>
-                {
-                    ["patterns"] = "**"
-                };
-
-                int exitCode = await processInvoker.ExecuteAsync(workingDirectory: fileFolder,
-                                              fileName: node,
-                                              arguments: $"\"{hashFilesScript.Replace("\"", "\\\"")}\"",
-                                              environment: env,
-                                              requireExitCodeZero: false,
-                                              outputEncoding: null,
-                                              killProcessOnCancel: true,
-                                              cancellationToken: token);
-
-                if (exitCode != 0)
-                {
-                    Trace.Error($"hashFiles returns '{exitCode}' failed. Fail to hash files under directory '{fileFolder}'");
-                }
-
-                stopWatch.Stop();
-                _updateTrace.Enqueue($"{nameof(HashFiles)}{Path.GetFileName(fileFolder)}Time: {stopWatch.ElapsedMilliseconds}ms");
-                return hashResult;
-            }
-        }
     }
 }

src/Runner.Listener/SelfUpdaterV2.cs

@@ -149,7 +149,6 @@ namespace GitHub.Runner.Listener
 
             string archiveFile = null;
 
-            // Only try trimmed package if sever sends them and we have calculated hash value of the current runtime/externals.
             _updateTrace.Enqueue($"DownloadUrl: {packageDownloadUrl}");
 
             try

src/Runner.Sdk/Util/IOUtil.cs

@@ -459,6 +459,34 @@ namespace GitHub.Runner.Sdk
             File.WriteAllText(path, null);
         }
 
+        /// <summary>
+        /// Replaces invalid file name characters with '_'
+        /// </summary>
+        public static string ReplaceInvalidFileNameChars(string fileName)
+        {
+            var result = new StringBuilder();
+            var invalidChars = Path.GetInvalidFileNameChars();
+
+            var current = 0; // Current index
+            while (current < fileName?.Length)
+            {
+                var next = fileName.IndexOfAny(invalidChars, current);
+                if (next >= 0)
+                {
+                    result.Append(fileName.Substring(current, next - current));
+                    result.Append('_');
+                    current = next + 1;
+                }
+                else
+                {
+                    result.Append(fileName.Substring(current));
+                    break;
+                }
+            }
+
+            return result.ToString();
+        }
+
         /// <summary>
         /// Recursively enumerates a directory without following directory reparse points.
         /// </summary>

src/Runner.Sdk/Util/VssUtil.cs

@@ -23,7 +23,13 @@ namespace GitHub.Runner.Sdk
 
             if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0)
             {
-                headerValues.AddRange(VssClientHttpRequestSettings.Default.UserAgent);
+                foreach (var headerVal in VssClientHttpRequestSettings.Default.UserAgent)
+                {
+                    if (!headerValues.Contains(headerVal))
+                    {
+                        headerValues.Add(headerVal);
+                    }
+                }
             }
 
             VssClientHttpRequestSettings.Default.UserAgent = headerValues;
@@ -33,6 +39,23 @@ namespace GitHub.Runner.Sdk
             {
                 VssClientHttpRequestSettings.Default.ServerCertificateValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
             }
+
+            var rawHeaderValues = new List<ProductInfoHeaderValue>();
+            rawHeaderValues.AddRange(additionalUserAgents);
+            rawHeaderValues.Add(new ProductInfoHeaderValue($"({StringUtil.SanitizeUserAgentHeader(RuntimeInformation.OSDescription)})"));
+
+            if (RawClientHttpRequestSettings.Default.UserAgent != null && RawClientHttpRequestSettings.Default.UserAgent.Count > 0)
+            {
+                foreach (var headerVal in RawClientHttpRequestSettings.Default.UserAgent)
+                {
+                    if (!rawHeaderValues.Contains(headerVal))
+                    {
+                        rawHeaderValues.Add(headerVal);
+                    }
+                }
+            }
+
+            RawClientHttpRequestSettings.Default.UserAgent = rawHeaderValues;
         }
 
         public static VssConnection CreateConnection(

src/Runner.Sdk/Util/WhichUtil.cs

@@ -7,129 +7,6 @@ namespace GitHub.Runner.Sdk
     public static class WhichUtil
     {
         public static string Which(string command, bool require = false, ITraceWriter trace = null, string prependPath = null)
-        {
-            ArgUtil.NotNullOrEmpty(command, nameof(command));
-            trace?.Info($"Which: '{command}'");
-            if (Path.IsPathFullyQualified(command) && File.Exists(command))
-            {
-                trace?.Info($"Fully qualified path: '{command}'");
-                return command;
-            }
-            string path = Environment.GetEnvironmentVariable(PathUtil.PathVariable);
-            if (string.IsNullOrEmpty(path))
-            {
-                trace?.Info("PATH environment variable not defined.");
-                path = path ?? string.Empty;
-            }
-            if (!string.IsNullOrEmpty(prependPath))
-            {
-                path = PathUtil.PrependPath(prependPath, path);
-            }
-
-            string[] pathSegments = path.Split(new Char[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries);
-            for (int i = 0; i < pathSegments.Length; i++)
-            {
-                pathSegments[i] = Environment.ExpandEnvironmentVariables(pathSegments[i]);
-            }
-
-            foreach (string pathSegment in pathSegments)
-            {
-                if (!string.IsNullOrEmpty(pathSegment) && Directory.Exists(pathSegment))
-                {
-                    string[] matches = null;
-#if OS_WINDOWS
-                    string pathExt = Environment.GetEnvironmentVariable("PATHEXT");
-                    if (string.IsNullOrEmpty(pathExt))
-                    {
-                        // XP's system default value for PATHEXT system variable
-                        pathExt = ".com;.exe;.bat;.cmd;.vbs;.vbe;.js;.jse;.wsf;.wsh";
-                    }
-
-                    string[] pathExtSegments = pathExt.Split(new string[] { ";" }, StringSplitOptions.RemoveEmptyEntries);
-
-                    // if command already has an extension.
-                    if (pathExtSegments.Any(ext => command.EndsWith(ext, StringComparison.OrdinalIgnoreCase)))
-                    {
-                        try
-                        {
-                            matches = Directory.GetFiles(pathSegment, command);
-                        }
-                        catch (UnauthorizedAccessException ex)
-                        {
-                            trace?.Info("Ignore UnauthorizedAccess exception during Which.");
-                            trace?.Verbose(ex.ToString());
-                        }
-
-                        if (matches != null && matches.Length > 0 && IsPathValid(matches.First(), trace))
-                        {
-                            trace?.Info($"Location: '{matches.First()}'");
-                            return matches.First();
-                        }
-                    }
-                    else
-                    {
-                        string searchPattern;
-                        searchPattern = StringUtil.Format($"{command}.*");
-                        try
-                        {
-                            matches = Directory.GetFiles(pathSegment, searchPattern);
-                        }
-                        catch (UnauthorizedAccessException ex)
-                        {
-                            trace?.Info("Ignore UnauthorizedAccess exception during Which.");
-                            trace?.Verbose(ex.ToString());
-                        }
-
-                        if (matches != null && matches.Length > 0)
-                        {
-                            // add extension.
-                            for (int i = 0; i < pathExtSegments.Length; i++)
-                            {
-                                string fullPath = Path.Combine(pathSegment, $"{command}{pathExtSegments[i]}");
-                                if (matches.Any(p => p.Equals(fullPath, StringComparison.OrdinalIgnoreCase)) && IsPathValid(fullPath, trace))
-                                {
-                                    trace?.Info($"Location: '{fullPath}'");
-                                    return fullPath;
-                                }
-                            }
-                        }
-                    }
-#else
-                    try
-                    {
-                        matches = Directory.GetFiles(pathSegment, command);
-                    }
-                    catch (UnauthorizedAccessException ex)
-                    {
-                        trace?.Info("Ignore UnauthorizedAccess exception during Which.");
-                        trace?.Verbose(ex.ToString());
-                    }
-
-                    if (matches != null && matches.Length > 0 && IsPathValid(matches.First(), trace))
-                    {
-                        trace?.Info($"Location: '{matches.First()}'");
-                        return matches.First();
-                    }
-#endif
-                }
-            }
-
-#if OS_WINDOWS
-            trace?.Info($"{command}: command not found. Make sure '{command}' is installed and its location included in the 'Path' environment variable.");
-#else
-            trace?.Info($"{command}: command not found. Make sure '{command}' is installed and its location included in the 'PATH' environment variable.");
-#endif
-            if (require)
-            {
-                throw new FileNotFoundException(
-                    message: $"{command}: command not found",
-                    fileName: command);
-            }
-
-            return null;
-        }
-
-        public static string Which2(string command, bool require = false, ITraceWriter trace = null, string prependPath = null)
         {
             ArgUtil.NotNullOrEmpty(command, nameof(command));
             trace?.Info($"Which2: '{command}'");

src/Runner.Worker/ActionManager.cs

@@ -483,10 +483,6 @@ namespace GitHub.Runner.Worker
                             {
                                 // Load stored Ids for later load actions
                                 compositeAction.Steps[i].Id = _cachedEmbeddedStepIds[action.Id][i];
-                                if (string.IsNullOrEmpty(executionContext.Global.Variables.Get("DistributedTask.EnableCompositeActions")) && compositeAction.Steps[i].Reference.Type != Pipelines.ActionSourceType.Script)
-                                {
-                                    throw new Exception("`uses:` keyword is not currently supported.");
-                                }
                             }
                         }
                         else
@@ -703,11 +699,12 @@ namespace GitHub.Runner.Worker
                 catch (Exception ex) when (!executionContext.CancellationToken.IsCancellationRequested) // Do not retry if the run is cancelled.
                 {
                     // UnresolvableActionDownloadInfoException is a 422 client error, don't retry
+                    // NonRetryableActionDownloadInfoException is an non-retryable exception from Actions
                     // Some possible cases are:
                     // * Repo is rate limited
                     // * Repo or tag doesn't exist, or isn't public
                     // * Policy validation failed
-                    if (attempt < 3 && !(ex is WebApi.UnresolvableActionDownloadInfoException))
+                    if (attempt < 3 && !(ex is WebApi.UnresolvableActionDownloadInfoException) && !(ex is WebApi.NonRetryableActionDownloadInfoException))
                     {
                         executionContext.Output($"Failed to resolve action download info. Error: {ex.Message}");
                         executionContext.Debug(ex.ToString());
@@ -796,43 +793,40 @@ namespace GitHub.Runner.Worker
             try
             {
                 var useActionArchiveCache = false;
-                if (executionContext.Global.Variables.GetBoolean("DistributedTask.UseActionArchiveCache") == true)
+                var hasActionArchiveCache = false;
+                var actionArchiveCacheDir = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory);
+                if (!string.IsNullOrEmpty(actionArchiveCacheDir) &&
+                    Directory.Exists(actionArchiveCacheDir))
                 {
-                    var hasActionArchiveCache = false;
-                    var actionArchiveCacheDir = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory);
-                    if (!string.IsNullOrEmpty(actionArchiveCacheDir) &&
-                        Directory.Exists(actionArchiveCacheDir))
-                    {
-                        hasActionArchiveCache = true;
-                        Trace.Info($"Check if action archive '{downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha}' already exists in cache directory '{actionArchiveCacheDir}'");
+                    hasActionArchiveCache = true;
+                    Trace.Info($"Check if action archive '{downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha}' already exists in cache directory '{actionArchiveCacheDir}'");
 #if OS_WINDOWS
-                        var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.zip");
+                    var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.zip");
 #else
-                        var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.tar.gz");
+                    var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.tar.gz");
 #endif
-                        if (File.Exists(cacheArchiveFile))
+                    if (File.Exists(cacheArchiveFile))
+                    {
+                        try
                         {
-                            try
-                            {
-                                Trace.Info($"Found action archive '{cacheArchiveFile}' in cache directory '{actionArchiveCacheDir}'");
-                                File.Copy(cacheArchiveFile, archiveFile);
-                                useActionArchiveCache = true;
-                                executionContext.Debug($"Copied action archive '{cacheArchiveFile}' to '{archiveFile}'");
-                            }
-                            catch (Exception ex)
-                            {
-                                Trace.Error($"Failed to copy action archive '{cacheArchiveFile}' to '{archiveFile}'. Error: {ex}");
-                            }
+                            Trace.Info($"Found action archive '{cacheArchiveFile}' in cache directory '{actionArchiveCacheDir}'");
+                            File.Copy(cacheArchiveFile, archiveFile);
+                            useActionArchiveCache = true;
+                            executionContext.Debug($"Copied action archive '{cacheArchiveFile}' to '{archiveFile}'");
+                        }
+                        catch (Exception ex)
+                        {
+                            Trace.Error($"Failed to copy action archive '{cacheArchiveFile}' to '{archiveFile}'. Error: {ex}");
                         }
                     }
-
-                    executionContext.Global.JobTelemetry.Add(new JobTelemetry()
-                    {
-                        Type = JobTelemetryType.General,
-                        Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache}"
-                    });
                 }
 
+                executionContext.Global.JobTelemetry.Add(new JobTelemetry()
+                {
+                    Type = JobTelemetryType.General,
+                    Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache}"
+                });
+
                 if (!useActionArchiveCache)
                 {
                     await DownloadRepositoryArchive(executionContext, link, downloadInfo.Authentication?.Token, archiveFile);
@@ -878,16 +872,9 @@ namespace GitHub.Runner.Worker
                     int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken);
                     if (exitCode != 0)
                     {
-                        if (executionContext.Global.Variables.GetBoolean("DistributedTask.DetailUntarFailure") == true)
-                        {
-                            var fileInfo = new FileInfo(archiveFile);
-                            var sha256hash = await IOUtil.GetFileContentSha256HashAsync(archiveFile);
-                            throw new InvalidActionArchiveException($"Can't use 'tar -xzf' extract archive file: {archiveFile} (SHA256 '{sha256hash}', size '{fileInfo.Length}' bytes, tar outputs '{string.Join(' ', tarOutputs)}'). Action being checked out: {downloadInfo.NameWithOwner}@{downloadInfo.Ref}. return code: {exitCode}.");
-                        }
-                        else
-                        {
-                            throw new InvalidActionArchiveException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. Action being checked out: {downloadInfo.NameWithOwner}@{downloadInfo.Ref}. return code: {exitCode}.");
-                        }
+                        var fileInfo = new FileInfo(archiveFile);
+                        var sha256hash = await IOUtil.GetFileContentSha256HashAsync(archiveFile);
+                        throw new InvalidActionArchiveException($"Can't use 'tar -xzf' extract archive file: {archiveFile} (SHA256 '{sha256hash}', size '{fileInfo.Length}' bytes, tar outputs '{string.Join(' ', tarOutputs)}'). Action being checked out: {downloadInfo.NameWithOwner}@{downloadInfo.Ref}. return code: {exitCode}.");
                     }
                 }
 #endif
@@ -1031,13 +1018,6 @@ namespace GitHub.Runner.Worker
                         }
                     }
 
-                    foreach (var step in compositeAction.Steps)
-                    {
-                        if (string.IsNullOrEmpty(executionContext.Global.Variables.Get("DistributedTask.EnableCompositeActions")) && step.Reference.Type != Pipelines.ActionSourceType.Script)
-                        {
-                            throw new Exception("`uses:` keyword is not currently supported.");
-                        }
-                    }
                     return setupInfo;
                 }
                 else
@@ -1122,6 +1102,7 @@ namespace GitHub.Runner.Worker
             int timeoutSeconds = 20 * 60;
             while (retryCount < 3)
             {
+                string requestId = string.Empty;
                 using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
                 using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken))
                 {
@@ -1137,7 +1118,7 @@ namespace GitHub.Runner.Worker
                             httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
                             using (var response = await httpClient.GetAsync(downloadUrl))
                             {
-                                var requestId = UrlUtil.GetGitHubRequestId(response.Headers);
+                                requestId = UrlUtil.GetGitHubRequestId(response.Headers);
                                 if (!string.IsNullOrEmpty(requestId))
                                 {
                                     Trace.Info($"Request URL: {downloadUrl} X-GitHub-Request-Id: {requestId} Http Status: {response.StatusCode}");
@@ -1175,7 +1156,7 @@ namespace GitHub.Runner.Worker
                     catch (OperationCanceledException ex) when (!executionContext.CancellationToken.IsCancellationRequested && retryCount >= 2)
                     {
                         Trace.Info($"Action download final retry timeout after {timeoutSeconds} seconds.");
-                        throw new TimeoutException($"Action '{downloadUrl}' download has timed out. Error: {ex.Message}");
+                        throw new TimeoutException($"Action '{downloadUrl}' download has timed out. Error: {ex.Message} {requestId}");
                     }
                     catch (ActionNotFoundException)
                     {
@@ -1190,11 +1171,11 @@ namespace GitHub.Runner.Worker
                         if (actionDownloadTimeout.Token.IsCancellationRequested)
                         {
                             // action download didn't finish within timeout
-                            executionContext.Warning($"Action '{downloadUrl}' didn't finish download within {timeoutSeconds} seconds.");
+                            executionContext.Warning($"Action '{downloadUrl}' didn't finish download within {timeoutSeconds} seconds. {requestId}");
                         }
                         else
                         {
-                            executionContext.Warning($"Failed to download action '{downloadUrl}'. Error: {ex.Message}");
+                            executionContext.Warning($"Failed to download action '{downloadUrl}'. Error: {ex.Message} {requestId}");
                         }
                     }
                 }

src/Runner.Worker/ActionManifestManager.cs

@@ -144,7 +144,7 @@ namespace GitHub.Runner.Worker
                     executionContext.Error(error.Message);
                 }
 
-                throw new ArgumentException($"Fail to load {fileRelativePath}");
+                throw new ArgumentException($"Failed to load {fileRelativePath}");
             }
 
             if (actionDefinition.Execution == null)

src/Runner.Worker/ContainerOperationProvider.cs

@@ -466,17 +466,39 @@ namespace GitHub.Runner.Worker
             {
                 throw new InvalidOperationException($"Failed to create directory to store registry client credentials: {e.Message}");
             }
-            var loginExitCode = await _dockerManager.DockerLogin(
-                executionContext,
-                configLocation,
-                container.RegistryServer,
-                container.RegistryAuthUsername,
-                container.RegistryAuthPassword);
 
-            if (loginExitCode != 0)
+            // Login docker with retry up to 3 times
+            int retryCount = 0;
+            int loginExitCode = 0;
+            while (retryCount < 3)
+            {
+                loginExitCode = await _dockerManager.DockerLogin(
+                    executionContext,
+                    configLocation,
+                    container.RegistryServer,
+                    container.RegistryAuthUsername,
+                    container.RegistryAuthPassword);
+                if (loginExitCode == 0)
+                {
+                    break;
+                }
+                else
+                {
+                    retryCount++;
+                    if (retryCount < 3)
+                    {
+                        var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
+                        executionContext.Warning($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
+                        await Task.Delay(backOff);
+                    }
+                }
+            }
+
+            if (retryCount == 3 && loginExitCode != 0)
             {
                 throw new InvalidOperationException($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}");
             }
+
             return configLocation;
         }
 

src/Runner.Worker/DiagnosticLogManager.cs

@@ -91,13 +91,13 @@ namespace GitHub.Runner.Worker
             string phaseName = executionContext.Global.Variables.System_PhaseDisplayName ?? "UnknownPhaseName";
 
             // zip the files
-            string diagnosticsZipFileName = $"{buildName}-{phaseName}.zip";
+            string diagnosticsZipFileName = $"{buildName}-{IOUtil.ReplaceInvalidFileNameChars(phaseName)}.zip";
             string diagnosticsZipFilePath = Path.Combine(supportRootFolder, diagnosticsZipFileName);
             ZipFile.CreateFromDirectory(supportFilesFolder, diagnosticsZipFilePath);
 
             // upload the json metadata file
             executionContext.Debug("Uploading diagnostic metadata file.");
-            string metadataFileName = $"diagnostics-{buildName}-{phaseName}.json";
+            string metadataFileName = $"diagnostics-{buildName}-{IOUtil.ReplaceInvalidFileNameChars(phaseName)}.json";
             string metadataFilePath = Path.Combine(supportFilesFolder, metadataFileName);
             string phaseResult = GetTaskResultAsString(executionContext.Result);
 
@@ -108,6 +108,8 @@ namespace GitHub.Runner.Worker
 
             parentContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath);
 
+            parentContext.QueueDiagnosticLogFile(name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath);
+
             executionContext.Debug("Diagnostic file upload complete.");
         }
 

src/Runner.Worker/ExecutionContext.cs

@@ -83,13 +83,14 @@ namespace GitHub.Runner.Worker
         // Initialize
         void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token);
         void CancelToken();
-        IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, ActionRunStage stage, Dictionary<string, string> intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool isEmbedded = false, CancellationTokenSource cancellationTokenSource = null, Guid embeddedId = default(Guid), string siblingScopeName = null, TimeSpan? timeout = null);
+        IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, ActionRunStage stage, Dictionary<string, string> intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool isEmbedded = false, List<Issue> embeddedIssueCollector = null, CancellationTokenSource cancellationTokenSource = null, Guid embeddedId = default(Guid), string siblingScopeName = null, TimeSpan? timeout = null);
         IExecutionContext CreateEmbeddedChild(string scopeName, string contextName, Guid embeddedId, ActionRunStage stage, Dictionary<string, string> intraActionState = null, string siblingScopeName = null);
 
         // logging
         long Write(string tag, string message);
         void QueueAttachFile(string type, string name, string filePath);
         void QueueSummaryFile(string name, string filePath, Guid stepRecordId);
+        void QueueDiagnosticLogFile(string name, string filePath);
 
         // timeline record update methods
         void Start(string currentOperation = null);
@@ -134,7 +135,6 @@ namespace GitHub.Runner.Worker
 
         private readonly TimelineRecord _record = new();
         private readonly Dictionary<Guid, TimelineRecord> _detailRecords = new();
-        private readonly List<Issue> _embeddedIssueCollector;
         private readonly object _loggerLock = new();
         private readonly object _matchersLock = new();
         private readonly ExecutionContext _parentExecutionContext;
@@ -153,6 +153,7 @@ namespace GitHub.Runner.Worker
         private CancellationTokenSource _cancellationTokenSource;
         private TaskCompletionSource<int> _forceCompleted = new();
         private bool _throttlingReported = false;
+        private List<Issue> _embeddedIssueCollector;
 
         // only job level ExecutionContext will track throttling delay.
         private long _totalThrottlingDelayInMilliseconds = 0;
@@ -355,6 +356,7 @@ namespace GitHub.Runner.Worker
             int? recordOrder = null,
             IPagingLogger logger = null,
             bool isEmbedded = false,
+            List<Issue> embeddedIssueCollector = null,
             CancellationTokenSource cancellationTokenSource = null,
             Guid embeddedId = default(Guid),
             string siblingScopeName = null,
@@ -364,6 +366,10 @@ namespace GitHub.Runner.Worker
 
             var child = new ExecutionContext(this, isEmbedded);
             child.Initialize(HostContext);
+            if ((Global.Variables.GetBoolean("RunService.FixEmbeddedIssues") ?? false) && embeddedIssueCollector != null)
+            {
+                child._embeddedIssueCollector = embeddedIssueCollector;
+            }
             child.Global = Global;
             child.ScopeName = scopeName;
             child.ContextName = contextName;
@@ -397,11 +403,11 @@ namespace GitHub.Runner.Worker
 
             if (recordOrder != null)
             {
-                child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, recordOrder);
+                child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, recordOrder, embedded: isEmbedded);
             }
             else
             {
-                child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, ++_childTimelineRecordOrder);
+                child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, ++_childTimelineRecordOrder, embedded: isEmbedded);
             }
             if (logger != null)
             {
@@ -432,7 +438,7 @@ namespace GitHub.Runner.Worker
             Dictionary<string, string> intraActionState = null,
             string siblingScopeName = null)
         {
-            return Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, contextName, stage, logger: _logger, isEmbedded: true, cancellationTokenSource: null, intraActionState: intraActionState, embeddedId: embeddedId, siblingScopeName: siblingScopeName, timeout: GetRemainingTimeout());
+            return Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, contextName, stage, logger: _logger, isEmbedded: true, embeddedIssueCollector: _embeddedIssueCollector, cancellationTokenSource: null, intraActionState: intraActionState, embeddedId: embeddedId, siblingScopeName: siblingScopeName, timeout: GetRemainingTimeout(), recordOrder: _record.Order);
         }
 
         public void Start(string currentOperation = null)
@@ -519,7 +525,6 @@ namespace GitHub.Runner.Worker
                 Global.StepsResult.Add(stepResult);
             }
 
-
             if (Root != this)
             {
                 // only dispose TokenSource for step level ExecutionContext
@@ -836,7 +841,6 @@ namespace GitHub.Runner.Worker
             // Actions environment
             ActionsEnvironment = message.ActionsEnvironment;
 
-
             // Service container info
             Global.ServiceContainers = new List<ContainerInfo>();
 
@@ -982,6 +986,18 @@ namespace GitHub.Runner.Worker
             _jobServerQueue.QueueResultsUpload(stepRecordId, name, filePath, ChecksAttachmentType.StepSummary, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0);
         }
 
+        public void QueueDiagnosticLogFile(string name, string filePath)
+        {
+            ArgUtil.NotNullOrEmpty(name, nameof(name));
+            ArgUtil.NotNullOrEmpty(filePath, nameof(filePath));
+
+            if (!File.Exists(filePath))
+            {
+                throw new FileNotFoundException($"Can't upload diagnostic log file: {filePath}. File does not exist.");
+            }
+            _jobServerQueue.QueueResultsUpload(_record.Id, name, filePath, CoreAttachmentType.ResultsDiagnosticLog, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0);
+        }
+
         // Add OnMatcherChanged
         public void Add(OnMatcherChanged handler)
         {
@@ -1160,7 +1176,7 @@ namespace GitHub.Runner.Worker
             }
         }
 
-        private void InitializeTimelineRecord(Guid timelineId, Guid timelineRecordId, Guid? parentTimelineRecordId, string recordType, string displayName, string refName, int? order)
+        private void InitializeTimelineRecord(Guid timelineId, Guid timelineRecordId, Guid? parentTimelineRecordId, string recordType, string displayName, string refName, int? order, bool embedded = false)
         {
             _mainTimelineId = timelineId;
             _record.Id = timelineRecordId;
@@ -1186,7 +1202,11 @@ namespace GitHub.Runner.Worker
             var configuration = HostContext.GetService<IConfigurationStore>();
             _record.WorkerName = configuration.GetSettings().AgentName;
 
-            _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
+            // We don't want to update the timeline record for embedded steps since they are not really represented in the UI.
+            if (!embedded)
+            {
+                _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
+            }
         }
 
         private void JobServerQueueThrottling_EventReceived(object sender, ThrottlingEventArgs data)
@@ -1401,7 +1421,7 @@ namespace GitHub.Runner.Worker
             {
                 if (key == PipelineTemplateConstants.HostWorkspace)
                 {
-                    // The HostWorkspace context var is excluded so that there is a var that always points to the host path. 
+                    // The HostWorkspace context var is excluded so that there is a var that always points to the host path.
                     // This var can be used to translate back from container paths, e.g. in HashFilesFunction, which always runs on the host machine
                     continue;
                 }

src/Runner.Worker/FileCommandManager.cs

@@ -244,7 +244,7 @@ namespace GitHub.Runner.Worker
                 if (resultsReceiverEndpoint != null)
                 {
                     Trace.Info($"Queueing results file ({filePath}) for attachment upload ({attachmentName})");
-                    var stepId = context.Id;
+                    var stepId = context.IsEmbedded ? context.EmbeddedId : context.Id;
                     // Attachments must be added to the parent context (job), not the current context (step)
                     context.Root.QueueSummaryFile(attachmentName, scrubbedFilePath, stepId);
                 }

src/Runner.Worker/Handlers/ContainerActionHandler.cs

@@ -223,6 +223,10 @@ namespace GitHub.Runner.Worker.Handlers
             {
                 Environment["ACTIONS_CACHE_URL"] = cacheUrl;
             }
+            if (systemConnection.Data.TryGetValue("PipelinesServiceUrl", out var pipelinesServiceUrl) && !string.IsNullOrEmpty(pipelinesServiceUrl))
+            {
+                Environment["ACTIONS_RUNTIME_URL"] = pipelinesServiceUrl;
+            }
             if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) && !string.IsNullOrEmpty(generateIdTokenUrl))
             {
                 Environment["ACTIONS_ID_TOKEN_REQUEST_URL"] = generateIdTokenUrl;

src/Runner.Worker/Handlers/HandlerFactory.cs

@@ -84,6 +84,45 @@ namespace GitHub.Runner.Worker.Handlers
                     }
                     nodeData.NodeVersion = "node16";
                 }
+
+                var localForceActionsToNode20 = StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Agent.ManualForceActionsToNode20));
+                executionContext.Global.EnvironmentVariables.TryGetValue(Constants.Variables.Actions.ManualForceActionsToNode20, out var workflowForceActionsToNode20);
+                var enforceNode20Locally = !string.IsNullOrWhiteSpace(workflowForceActionsToNode20) ? StringUtil.ConvertToBoolean(workflowForceActionsToNode20) : localForceActionsToNode20;
+                if (string.Equals(nodeData.NodeVersion, "node16")
+                && ((executionContext.Global.Variables.GetBoolean("DistributedTask.ForceGithubJavascriptActionsToNode20") ?? false) || enforceNode20Locally))
+                {
+                    executionContext.Global.EnvironmentVariables.TryGetValue(Constants.Variables.Actions.AllowActionsUseUnsecureNodeVersion, out var workflowOptOut);
+                    var isWorkflowOptOutSet = !string.IsNullOrWhiteSpace(workflowOptOut);
+                    var isLocalOptOut = StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Actions.AllowActionsUseUnsecureNodeVersion));
+                    bool isOptOut = isWorkflowOptOutSet ? StringUtil.ConvertToBoolean(workflowOptOut) : isLocalOptOut;
+
+                    if (!isOptOut)
+                    {
+                        var repoAction = action as Pipelines.RepositoryPathReference;
+                        if (repoAction != null)
+                        {
+                            var warningActions = new HashSet<string>();
+                            if (executionContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings))
+                            {
+                                warningActions = StringUtil.ConvertFromJson<HashSet<string>>(node20ForceWarnings);
+                            }
+
+                            string repoActionFullName;
+                            if (string.IsNullOrEmpty(repoAction.Name))
+                            {
+                                repoActionFullName = repoAction.Path; // local actions don't have a 'Name'
+                            }
+                            else
+                            {
+                                repoActionFullName = $"{repoAction.Name}/{repoAction.Path ?? string.Empty}".TrimEnd('/') + $"@{repoAction.Ref}";
+                            }
+
+                            warningActions.Add(repoActionFullName);
+                            executionContext.Global.Variables.Set(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, StringUtil.ConvertToJson(warningActions));
+                        }
+                        nodeData.NodeVersion = "node20";
+                    }
+                }
                 (handler as INodeScriptActionHandler).Data = nodeData;
             }
             else if (data.ExecutionType == ActionExecutionType.Script)

src/Runner.Worker/Handlers/NodeScriptActionHandler.cs

@@ -58,6 +58,10 @@ namespace GitHub.Runner.Worker.Handlers
             {
                 Environment["ACTIONS_CACHE_URL"] = cacheUrl;
             }
+            if (systemConnection.Data.TryGetValue("PipelinesServiceUrl", out var pipelinesServiceUrl) && !string.IsNullOrEmpty(pipelinesServiceUrl))
+            {
+                Environment["ACTIONS_RUNTIME_URL"] = pipelinesServiceUrl;
+            }
             if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) && !string.IsNullOrEmpty(generateIdTokenUrl))
             {
                 Environment["ACTIONS_ID_TOKEN_REQUEST_URL"] = generateIdTokenUrl;
@@ -89,7 +93,6 @@ namespace GitHub.Runner.Worker.Handlers
                 ExecutionContext.StepTelemetry.HasPreStep = Data.HasPre;
                 ExecutionContext.StepTelemetry.HasPostStep = Data.HasPost;
             }
-            ExecutionContext.StepTelemetry.Type = Data.NodeVersion;
 
             ArgUtil.NotNullOrEmpty(target, nameof(target));
             target = Path.Combine(ActionDirectory, target);
@@ -114,7 +117,13 @@ namespace GitHub.Runner.Worker.Handlers
             {
                 Data.NodeVersion = "node16";
             }
+
+            if (forcedNodeVersion == "node20" && Data.NodeVersion != "node20")
+            {
+                Data.NodeVersion = "node20";
+            }
             var nodeRuntimeVersion = await StepHost.DetermineNodeRuntimeVersion(ExecutionContext, Data.NodeVersion);
+            ExecutionContext.StepTelemetry.Type = nodeRuntimeVersion;
             string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), nodeRuntimeVersion, "bin", $"node{IOUtil.ExeExtension}");
 
             // Format the arguments passed to node.

src/Runner.Worker/Handlers/ScriptHandler.cs

@@ -83,40 +83,19 @@ namespace GitHub.Runner.Worker.Handlers
                 shellCommand = "pwsh";
                 if (validateShellOnHost)
                 {
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        shellCommandPath = WhichUtil.Which2(shellCommand, require: false, Trace, prependPath);
-                    }
-                    else
-                    {
-                        shellCommandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
-                    }
+                    shellCommandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
                     if (string.IsNullOrEmpty(shellCommandPath))
                     {
                         shellCommand = "powershell";
-                        Trace.Info($"Defaulting to {shellCommand}");
-                        if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                        {
-                            shellCommandPath = WhichUtil.Which2(shellCommand, require: true, Trace, prependPath);
-                        }
-                        else
-                        {
-                            shellCommandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath);
-                        }
+                        Trace.Info($"Defaulting to {shellCommand}");                        
+                        shellCommandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath);
                     }
                 }
 #else
                 shellCommand = "sh";
                 if (validateShellOnHost)
                 {
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        shellCommandPath = WhichUtil.Which2("bash", false, Trace, prependPath) ?? WhichUtil.Which2("sh", true, Trace, prependPath);
-                    }
-                    else
-                    {
-                        shellCommandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath);
-                    }
+                    shellCommandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath);
                 }
 #endif
                 argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand);
@@ -127,14 +106,7 @@ namespace GitHub.Runner.Worker.Handlers
                 shellCommand = parsed.shellCommand;
                 if (validateShellOnHost)
                 {
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        shellCommandPath = WhichUtil.Which2(parsed.shellCommand, true, Trace, prependPath);
-                    }
-                    else
-                    {
-                        shellCommandPath = WhichUtil.Which(parsed.shellCommand, true, Trace, prependPath);
-                    }
+                    shellCommandPath = WhichUtil.Which(parsed.shellCommand, true, Trace, prependPath);
                 }
 
                 argFormat = $"{parsed.shellArgs}".TrimStart();
@@ -216,38 +188,17 @@ namespace GitHub.Runner.Worker.Handlers
             {
 #if OS_WINDOWS
                 shellCommand = "pwsh";
-                if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                {
-                    commandPath = WhichUtil.Which2(shellCommand, require: false, Trace, prependPath);
-                }
-                else
-                {
-                    commandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
-                }
+                commandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
                 if (string.IsNullOrEmpty(commandPath))
                 {
                     shellCommand = "powershell";
                     Trace.Info($"Defaulting to {shellCommand}");
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        commandPath = WhichUtil.Which2(shellCommand, require: true, Trace, prependPath);
-                    }
-                    else
-                    {
-                        commandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath);
-                    }
+                    commandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath);
                 }
                 ArgUtil.NotNullOrEmpty(commandPath, "Default Shell");
 #else
                 shellCommand = "sh";
-                if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                {
-                    commandPath = WhichUtil.Which2("bash", false, Trace, prependPath) ?? WhichUtil.Which2("sh", true, Trace, prependPath);
-                }
-                else
-                {
-                    commandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath);
-                }
+                commandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath);
 #endif
                 argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand);
             }
@@ -258,14 +209,7 @@ namespace GitHub.Runner.Worker.Handlers
                 if (!IsActionStep && systemShells.Contains(shell))
                 {
                     shellCommand = shell;
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        commandPath = WhichUtil.Which2(shell, !isContainerStepHost, Trace, prependPath);
-                    }
-                    else
-                    {
-                        commandPath = WhichUtil.Which(shell, !isContainerStepHost, Trace, prependPath);
-                    }
+                    commandPath = WhichUtil.Which(shell, !isContainerStepHost, Trace, prependPath);
                     if (shell == "bash")
                     {
                         argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat("sh");
@@ -280,14 +224,7 @@ namespace GitHub.Runner.Worker.Handlers
                     var parsed = ScriptHandlerHelpers.ParseShellOptionString(shell);
                     shellCommand = parsed.shellCommand;
                     // For non-ContainerStepHost, the command must be located on the host by Which
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        commandPath = WhichUtil.Which2(parsed.shellCommand, !isContainerStepHost, Trace, prependPath);
-                    }
-                    else
-                    {
-                        commandPath = WhichUtil.Which(parsed.shellCommand, !isContainerStepHost, Trace, prependPath);
-                    }
+                    commandPath = WhichUtil.Which(parsed.shellCommand, !isContainerStepHost, Trace, prependPath);
                     argFormat = $"{parsed.shellArgs}".TrimStart();
                     if (string.IsNullOrEmpty(argFormat))
                     {

src/Runner.Worker/JobExtension.cs

@@ -127,6 +127,10 @@ namespace GitHub.Runner.Worker
                         }
                     }
 
+                    // Check OS warning
+                    var osWarningChecker = HostContext.GetService<IOSWarningChecker>();
+                    await osWarningChecker.CheckOSAsync(context);
+
                     try
                     {
                         var tokenPermissions = jobContext.Global.Variables.Get("system.github.token.permissions") ?? "";
@@ -392,6 +396,18 @@ namespace GitHub.Runner.Worker
                         }
                     }
 
+                    // Register custom image creation post-job step if the "snapshot" token is present in the message.
+                    var snapshotRequest = templateEvaluator.EvaluateJobSnapshotRequest(message.Snapshot, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
+                    if (snapshotRequest != null)
+                    {
+                        var snapshotOperationProvider = HostContext.GetService<ISnapshotOperationProvider>();
+                        jobContext.RegisterPostJobStep(new JobExtensionRunner(
+                            runAsync: (executionContext, _) => snapshotOperationProvider.CreateSnapshotRequestAsync(executionContext, snapshotRequest),
+                            condition: snapshotRequest.Condition,
+                            displayName: $"Create custom image",
+                            data: null));
+                    }
+
                     // Register Job Completed hook if the variable is set
                     var completedHookPath = Environment.GetEnvironmentVariable("ACTIONS_RUNNER_HOOK_JOB_COMPLETED");
                     if (!string.IsNullOrEmpty(completedHookPath))

src/Runner.Worker/JobRunner.cs

@@ -42,6 +42,7 @@ namespace GitHub.Runner.Worker
             Trace.Info("Job ID {0}", message.JobId);
 
             DateTime jobStartTimeUtc = DateTime.UtcNow;
+            _runnerSettings = HostContext.GetService<IConfigurationStore>().GetSettings();
             IRunnerService server = null;
 
             // add orchestration id to useragent for better correlation.
@@ -49,13 +50,9 @@ namespace GitHub.Runner.Worker
                 !string.IsNullOrEmpty(orchestrationId.Value))
             {
                 HostContext.UserAgents.Add(new ProductInfoHeaderValue("OrchestrationId", orchestrationId.Value));
-            }
 
-            var jobServerQueueTelemetry = false;
-            if (message.Variables.TryGetValue("DistributedTask.EnableJobServerQueueTelemetry", out VariableValue enableJobServerQueueTelemetry) &&
-                !string.IsNullOrEmpty(enableJobServerQueueTelemetry?.Value))
-            {
-                jobServerQueueTelemetry = StringUtil.ConvertToBoolean(enableJobServerQueueTelemetry.Value);
+                // make sure orchestration id is in the user-agent header.
+                VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy);
             }
 
             ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
@@ -79,7 +76,7 @@ namespace GitHub.Runner.Worker
                     launchServer.InitializeLaunchClient(new Uri(launchReceiverEndpoint), accessToken);
                 }
                 _jobServerQueue = HostContext.GetService<IJobServerQueue>();
-                _jobServerQueue.Start(message, resultsServiceOnly: true, enableTelemetry: jobServerQueueTelemetry);
+                _jobServerQueue.Start(message, resultsServiceOnly: true);
             }
             else
             {
@@ -101,7 +98,7 @@ namespace GitHub.Runner.Worker
                 VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, delegatingHandlers);
                 await jobServer.ConnectAsync(jobConnection);
 
-                _jobServerQueue.Start(message, enableTelemetry: jobServerQueueTelemetry);
+                _jobServerQueue.Start(message);
                 server = jobServer;
             }
 
@@ -161,8 +158,6 @@ namespace GitHub.Runner.Worker
 
                 jobContext.SetRunnerContext("os", VarUtil.OS);
                 jobContext.SetRunnerContext("arch", VarUtil.OSArchitecture);
-
-                _runnerSettings = HostContext.GetService<IConfigurationStore>().GetSettings();
                 jobContext.SetRunnerContext("name", _runnerSettings.AgentName);
 
                 if (jobContext.Global.Variables.TryGetValue(WellKnownDistributedTaskVariables.RunnerEnvironment, out var runnerEnvironment))
@@ -295,6 +290,14 @@ namespace GitHub.Runner.Worker
                 jobContext.Warning(string.Format(Constants.Runner.EnforcedNode12DetectedAfterEndOfLife, actions));
             }
 
+            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings) && (jobContext.Global.Variables.GetBoolean("DistributedTask.ForceGithubJavascriptActionsToNode20") ?? false))
+            {
+                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(node20ForceWarnings));
+                jobContext.Warning(string.Format(Constants.Runner.EnforcedNode16DetectedAfterEndOfLife, actions));
+            }
+
+            await ShutdownQueue(throwOnFailure: false);
+
             // Make sure to clean temp after file upload since they may be pending fileupload still use the TEMP dir.
             _tempDirectoryManager?.CleanupTempDirectory();
 
@@ -400,6 +403,12 @@ namespace GitHub.Runner.Worker
                 jobContext.Warning(string.Format(Constants.Runner.EnforcedNode12DetectedAfterEndOfLife, actions));
             }
 
+            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings))
+            {
+                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(node20ForceWarnings));
+                jobContext.Warning(string.Format(Constants.Runner.EnforcedNode16DetectedAfterEndOfLife, actions));
+            }
+
             try
             {
                 var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: true);

src/Runner.Worker/OSWarningChecker.cs

@@ -0,0 +1,110 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Common;
+using GitHub.Runner.Sdk;
+
+namespace GitHub.Runner.Worker
+{
+    [ServiceLocator(Default = typeof(OSWarningChecker))]
+    public interface IOSWarningChecker : IRunnerService
+    {
+        Task CheckOSAsync(IExecutionContext context);
+    }
+
+    public sealed class OSWarningChecker : RunnerService, IOSWarningChecker
+    {
+        private static TimeSpan s_regexTimeout = TimeSpan.FromSeconds(1);
+
+        public async Task CheckOSAsync(IExecutionContext context)
+        {
+            ArgUtil.NotNull(context, nameof(context));
+            if (!context.Global.Variables.System_TestDotNet8Compatibility)
+            {
+                return;
+            }
+
+            context.Output("Testing runner upgrade compatibility");
+            List<string> output = new();
+            object outputLock = new();
+            try
+            {
+                using (var process = HostContext.CreateService<IProcessInvoker>())
+                {
+                    process.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stdout)
+                    {
+                        if (!string.IsNullOrEmpty(stdout.Data))
+                        {
+                            lock (outputLock)
+                            {
+                                output.Add(stdout.Data);
+                                Trace.Info(stdout.Data);
+                            }
+                        }
+                    };
+
+                    process.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stderr)
+                    {
+                        if (!string.IsNullOrEmpty(stderr.Data))
+                        {
+                            lock (outputLock)
+                            {
+                                output.Add(stderr.Data);
+                                Trace.Error(stderr.Data);
+                            }
+                        }
+                    };
+
+                    using (var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
+                    {
+                        int exitCode = await process.ExecuteAsync(
+                            workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root),
+                            fileName: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "testDotNet8Compatibility", $"TestDotNet8Compatibility{IOUtil.ExeExtension}"),
+                            arguments: string.Empty,
+                            environment: null,
+                            cancellationToken: cancellationTokenSource.Token);
+
+                        var outputStr = string.Join("\n", output).Trim();
+                        if (exitCode != 0 || !string.Equals(outputStr, "Hello from .NET 8!", StringComparison.Ordinal))
+                        {
+                            var pattern = context.Global.Variables.System_DotNet8CompatibilityOutputPattern;
+                            if (!string.IsNullOrEmpty(pattern))
+                            {
+                                var regex = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, s_regexTimeout);
+                                if (!regex.IsMatch(outputStr))
+                                {
+                                    return;
+                                }
+                            }
+
+                            var warningMessage = context.Global.Variables.System_DotNet8CompatibilityWarning;
+                            if (!string.IsNullOrEmpty(warningMessage))
+                            {
+                                context.Warning(warningMessage);
+                            }
+
+                            context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test failed with exit code '{exitCode}' and output: {GetShortOutput(context, output)}" });
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Trace.Error("An error occurred while testing .NET 8 compatibility'");
+                Trace.Error(ex);
+                context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test encountered exception type '{ex.GetType().FullName}', message: '{ex.Message}', process output: '{GetShortOutput(context, output)}'" });
+            }
+        }
+
+        private static string GetShortOutput(IExecutionContext context, List<string> output)
+        {
+            var length = context.Global.Variables.System_DotNet8CompatibilityOutputLength ?? 200;
+            var outputStr = string.Join("\n", output).Trim();
+            return outputStr.Length > length ? string.Concat(outputStr.Substring(0, length), "[...]") : outputStr;
+        }
+    }
+}

src/Runner.Worker/SnapshotOperationProvider.cs

@@ -0,0 +1,32 @@
+#nullable enable
+using System.IO;
+using System.Threading.Tasks;
+using GitHub.DistributedTask.Pipelines;
+using GitHub.Runner.Common;
+using GitHub.Runner.Sdk;
+namespace GitHub.Runner.Worker;
+
+[ServiceLocator(Default = typeof(SnapshotOperationProvider))]
+public interface ISnapshotOperationProvider : IRunnerService
+{
+    Task CreateSnapshotRequestAsync(IExecutionContext executionContext, Snapshot snapshotRequest);
+}
+
+public class SnapshotOperationProvider : RunnerService, ISnapshotOperationProvider
+{
+    public Task CreateSnapshotRequestAsync(IExecutionContext executionContext, Snapshot snapshotRequest)
+    {
+        var snapshotRequestFilePath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), ".snapshot", "request.json");
+        var snapshotRequestDirectoryPath = Path.GetDirectoryName(snapshotRequestFilePath);
+        if (snapshotRequestDirectoryPath != null)
+        {
+            Directory.CreateDirectory(snapshotRequestDirectoryPath);
+        }
+
+        IOUtil.SaveObject(snapshotRequest, snapshotRequestFilePath);
+        executionContext.Output($"Request written to: {snapshotRequestFilePath}");
+        executionContext.Output("This request will be processed after the job completes. You will not receive any feedback on the snapshot process within the workflow logs of this job.");
+        executionContext.Output("If the snapshot process is successful, you should see a new image with the requested name in the list of available custom images when creating a new GitHub-hosted Runner.");
+        return Task.CompletedTask;
+    }
+}

src/Runner.Worker/StepsRunner.cs

@@ -295,7 +295,7 @@ namespace GitHub.Runner.Worker
                     !jobCancellationToken.IsCancellationRequested)
                 {
                     Trace.Error($"Caught timeout exception from step: {ex.Message}");
-                    step.ExecutionContext.Error("The action has timed out.");
+                    step.ExecutionContext.Error($"The action '{step.DisplayName}' has timed out after {timeoutMinutes} minutes.");
                     step.ExecutionContext.Result = TaskResult.Failed;
                 }
                 else

src/Runner.Worker/Variables.cs

@@ -72,8 +72,16 @@ namespace GitHub.Runner.Worker
 
         public bool? Step_Debug => GetBoolean(Constants.Variables.Actions.StepDebug);
 
+        public string System_DotNet8CompatibilityWarning => Get(Constants.Variables.System.DotNet8CompatibilityWarning);
+
+        public string System_DotNet8CompatibilityOutputPattern => Get(Constants.Variables.System.DotNet8CompatibilityOutputPattern);
+
+        public int? System_DotNet8CompatibilityOutputLength => GetInt(Constants.Variables.System.DotNet8CompatibilityOutputLength);
+
         public string System_PhaseDisplayName => Get(Constants.Variables.System.PhaseDisplayName);
 
+        public bool System_TestDotNet8Compatibility => GetBoolean(Constants.Variables.System.TestDotNet8Compatibility) ?? false;
+
         public string Get(string name)
         {
             Variable variable;

src/Sdk/Common/Common/RawHttpHeaders.cs

@@ -1,12 +1,11 @@
 using System;
 using System.ComponentModel;
-using System.Diagnostics.CodeAnalysis;
 
 namespace GitHub.Services.Common.Internal
 {
     [EditorBrowsable(EditorBrowsableState.Never)]
     public static class RawHttpHeaders
     {
-        public const String SessionHeader = "X-Runner-Session";
+        public const String SessionHeader = "X-Actions-Session";
     }
 }

src/Sdk/Common/Common/RawHttpMessageHandler.cs

@@ -138,6 +138,8 @@ namespace GitHub.Services.Common
                         response.Dispose();
                     }
 
+                    this.Settings.ApplyTo(request);
+
                     // Let's start with sending a token
                     IssuedToken token = null;
                     if (m_tokenProvider != null)

src/Sdk/Common/Common/VssHttpMessageHandler.cs

@@ -214,25 +214,7 @@ namespace GitHub.Services.Common
                     // ConfigureAwait(false) enables the continuation to be run outside any captured
                     // SyncronizationContext (such as ASP.NET's) which keeps things from deadlocking...
 
-                    var tmpResponse = await m_messageInvoker.SendAsync(request, tokenSource.Token).ConfigureAwait(false);
-                    if (tmpResponse.StatusCode == HttpStatusCode.Redirect)
-                    {
-                        //Dispose of the previous response
-                        tmpResponse?.Dispose();
-
-                        var location = tmpResponse.Headers.Location;
-                        request = new HttpRequestMessage(HttpMethod.Get, location);
-
-                        // Reapply the token to new redirected request
-                        ApplyToken(request, token, applyICredentialsToWebProxy: lastResponseDemandedProxyAuth);
-
-                        // Resend the request
-                        response = await m_messageInvoker.SendAsync(request, tokenSource.Token).ConfigureAwait(false);
-                    }
-                    else
-                    {
-                        response = tmpResponse;
-                    }
+                    response = await m_messageInvoker.SendAsync(request, tokenSource.Token).ConfigureAwait(false);
 
                     traceInfo?.TraceRequestSendTime();
 

src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs

@@ -43,6 +43,7 @@ namespace GitHub.DistributedTask.Pipelines
             TemplateToken jobOutputs,
             IList<TemplateToken> defaults,
             ActionsEnvironmentReference actionsEnvironment,
+            TemplateToken snapshot,
             String messageType = JobRequestMessageTypes.PipelineAgentJobRequest)
         {
             this.MessageType = messageType;
@@ -57,6 +58,7 @@ namespace GitHub.DistributedTask.Pipelines
             this.Workspace = workspaceOptions;
             this.JobOutputs = jobOutputs;
             this.ActionsEnvironment = actionsEnvironment;
+            this.Snapshot = snapshot;
             m_variables = new Dictionary<String, VariableValue>(variables, StringComparer.OrdinalIgnoreCase);
             m_maskHints = new List<MaskHint>(maskHints);
             m_steps = new List<JobStep>(steps);
@@ -237,6 +239,13 @@ namespace GitHub.DistributedTask.Pipelines
             set;
         }
 
+        [DataMember(EmitDefaultValue = false)]
+        public TemplateToken Snapshot
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Gets the collection of variables associated with the current context.
         /// </summary>

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs

@@ -29,6 +29,8 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
         public const String Id = "id";
         public const String If = "if";
         public const String Image = "image";
+        public const String ImageName = "image-name";
+        public const String CustomImageVersion = "version";
         public const String Include = "include";
         public const String Inputs = "inputs";
         public const String Job = "job";
@@ -60,6 +62,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
         public const String Services = "services";
         public const String Shell = "shell";
         public const String Skipped = "skipped";
+        public const String Snapshot = "snapshot";
         public const String StepEnv = "step-env";
         public const String StepIfResult = "step-if-result";
         public const String StepWith = "step-with";

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.ComponentModel;
+using System.Globalization;
 using System.Linq;
 using GitHub.DistributedTask.Expressions2;
 using GitHub.DistributedTask.Expressions2.Sdk;
@@ -346,6 +347,70 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
             return result;
         }
 
+        internal static Snapshot ConvertToJobSnapshotRequest(TemplateContext context, TemplateToken token)
+        {
+            string imageName = null;
+            string version = "1.*";
+            string versionString = string.Empty;
+            var condition = $"{PipelineTemplateConstants.Success}()";
+
+            if (token is StringToken snapshotStringLiteral)
+            {
+                imageName = snapshotStringLiteral.Value;
+            }
+            else
+            {
+                var snapshotMapping = token.AssertMapping($"{PipelineTemplateConstants.Snapshot}");
+                foreach (var snapshotPropertyPair in snapshotMapping)
+                {
+                    var propertyName = snapshotPropertyPair.Key.AssertString($"{PipelineTemplateConstants.Snapshot} key");
+                    var propertyValue = snapshotPropertyPair.Value;
+                    switch (propertyName.Value)
+                    {
+                        case PipelineTemplateConstants.ImageName:
+                            imageName = snapshotPropertyPair.Value.AssertString($"{PipelineTemplateConstants.Snapshot} {propertyName}").Value;
+                            break;
+                        case PipelineTemplateConstants.If:
+                            condition = ConvertToIfCondition(context, propertyValue, false);
+                            break;
+                        case PipelineTemplateConstants.CustomImageVersion:
+                            versionString = propertyValue.AssertString($"job {PipelineTemplateConstants.Snapshot} {PipelineTemplateConstants.CustomImageVersion}").Value;
+                            version = IsSnapshotImageVersionValid(versionString) ? versionString : null;
+                            break;
+                        default:
+                            propertyName.AssertUnexpectedValue($"{PipelineTemplateConstants.Snapshot} key");
+                            break;
+                    }
+                }
+            }
+
+            if (String.IsNullOrEmpty(imageName))
+            {
+                return null;
+            }
+
+            return new Snapshot(imageName)
+            {
+                Condition = condition,
+                Version = version
+            };
+        }
+
+        private static bool IsSnapshotImageVersionValid(string versionString)
+        {
+            var versionSegments = versionString.Split(".");
+
+            if (versionSegments.Length != 2 ||
+                !versionSegments[1].Equals("*") ||
+                !Int32.TryParse(versionSegments[0], NumberStyles.None, CultureInfo.InvariantCulture, result: out int parsedMajor) ||
+                parsedMajor < 0)
+            {
+                return false;
+            }
+
+            return true;
+        }
+
         private static ActionStep ConvertToStep(
             TemplateContext context,
             TemplateToken stepsItem,

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs

@@ -370,6 +370,32 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
             return result;
         }
 
+        public Snapshot EvaluateJobSnapshotRequest(TemplateToken token,
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions)
+        {
+            var result = default(Snapshot);
+
+            if (token != null && token.Type != TokenType.Null)
+            {
+                var context = CreateContext(contextData, expressionFunctions);
+                try
+                {
+                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.Snapshot, token, 0, null, omitHeader: true);
+                    context.Errors.Check();
+                    result = PipelineTemplateConverter.ConvertToJobSnapshotRequest(context, token);
+                }
+                catch (Exception ex) when (!(ex is TemplateValidationException))
+                {
+                    context.Errors.Add(ex);
+                }
+
+                context.Errors.Check();
+            }
+
+            return result;
+        }
+
         private TemplateContext CreateContext(
             DictionaryContextData contextData,
             IList<IFunctionInfo> expressionFunctions,

src/Sdk/DTPipelines/Pipelines/Snapshot.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Runtime.Serialization;
+using GitHub.DistributedTask.ObjectTemplating.Tokens;
+using GitHub.DistributedTask.Pipelines.ObjectTemplating;
+
+namespace GitHub.DistributedTask.Pipelines
+{
+    [DataContract]
+    public class Snapshot
+    {
+        public Snapshot(string imageName, string condition = null, string version = null)
+        {
+            ImageName = imageName;
+            Condition = condition ?? $"{PipelineTemplateConstants.Success}()";
+            Version = version ?? "1.*";
+        }
+
+        [DataMember(EmitDefaultValue = false)]
+        public String ImageName { get; set; }
+
+        [DataMember(EmitDefaultValue = false)]
+        public String Condition { get; set; }
+
+        [DataMember(EmitDefaultValue = false)]
+        public String Version { get; set; }
+    }
+}

src/Sdk/DTPipelines/workflow-v1.0.json

@@ -71,7 +71,8 @@
           "env": "job-env",
           "outputs": "job-outputs",
           "defaults": "job-defaults",
-          "steps": "steps"
+          "steps": "steps",
+          "snapshot": "snapshot"
         }
       }
     },
@@ -155,6 +156,41 @@
       }
     },
 
+    "snapshot": {
+      "one-of": [
+        "non-empty-string",
+        "snapshot-mapping"
+      ]
+    },
+
+    "snapshot-mapping": {
+      "mapping": {
+        "properties": {
+          "image-name": {
+            "type": "non-empty-string",
+            "required": true
+          },
+          "if": "snapshot-if",
+          "version": {
+            "type": "non-empty-string",
+            "required": false
+          }
+        }
+      }
+    },
+
+    "snapshot-if": {
+      "context": [
+        "github",
+        "inputs",
+        "vars",
+        "needs",
+        "strategy",
+        "matrix"
+      ],
+      "string": {}
+    },
+
     "runs-on": {
       "context": [
         "github",

src/Sdk/DTWebApi/WebApi/ActionsRunServerHttpClient.cs

@@ -0,0 +1,95 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.Services.Common;
+using GitHub.Services.Common.Diagnostics;
+using GitHub.Services.WebApi;
+using Newtonsoft.Json;
+
+namespace GitHub.DistributedTask.WebApi
+{
+    [ResourceArea(TaskResourceIds.AreaId)]
+    public class ActionsRunServerHttpClient : TaskAgentHttpClient
+    {
+        private static readonly JsonSerializerSettings s_serializerSettings;
+
+        static ActionsRunServerHttpClient()
+        {
+            s_serializerSettings = new VssJsonMediaTypeFormatter().SerializerSettings;
+            s_serializerSettings.DateParseHandling = DateParseHandling.None;
+            s_serializerSettings.FloatParseHandling = FloatParseHandling.Double;
+        }
+
+        public ActionsRunServerHttpClient(
+            Uri baseUrl,
+            VssCredentials credentials)
+            : base(baseUrl, credentials)
+        {
+        }
+
+        public ActionsRunServerHttpClient(
+            Uri baseUrl,
+            VssCredentials credentials,
+            VssHttpRequestSettings settings)
+            : base(baseUrl, credentials, settings)
+        {
+        }
+
+        public ActionsRunServerHttpClient(
+            Uri baseUrl,
+            VssCredentials credentials,
+            params DelegatingHandler[] handlers)
+            : base(baseUrl, credentials, handlers)
+        {
+        }
+
+        public ActionsRunServerHttpClient(
+            Uri baseUrl,
+            VssCredentials credentials,
+            VssHttpRequestSettings settings,
+            params DelegatingHandler[] handlers)
+            : base(baseUrl, credentials, settings, handlers)
+        {
+        }
+
+        public ActionsRunServerHttpClient(
+            Uri baseUrl,
+            HttpMessageHandler pipeline,
+            Boolean disposeHandler)
+            : base(baseUrl, pipeline, disposeHandler)
+        {
+        }
+
+        public Task<Pipelines.AgentJobRequestMessage> GetJobMessageAsync(
+            string messageId,
+            object userState = null,
+            CancellationToken cancellationToken = default)
+        {
+            HttpMethod httpMethod = new HttpMethod("GET");
+            Guid locationId = new Guid("25adab70-1379-4186-be8e-b643061ebe3a");
+            object routeValues = new { messageId = messageId };
+
+            return SendAsync<Pipelines.AgentJobRequestMessage>(
+                httpMethod,
+                locationId,
+                routeValues: routeValues,
+                version: new ApiResourceVersion(6.0, 1),
+                userState: userState,
+                cancellationToken: cancellationToken);
+        }
+
+        protected override async Task<T> ReadJsonContentAsync<T>(HttpResponseMessage response, CancellationToken cancellationToken = default(CancellationToken))
+        {
+            var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
+            return JsonConvert.DeserializeObject<T>(json, s_serializerSettings);
+        }
+    }
+}

src/Sdk/DTWebApi/WebApi/BrokerMigrationMessage.cs

@@ -0,0 +1,38 @@
+using System;
+using System.Runtime.Serialization;
+
+namespace GitHub.DistributedTask.WebApi
+{
+    /// <summary>
+    /// Message that tells the runner to redirect itself to BrokerListener for messages.
+    /// (Note that we use a special Message instead of a simple 302. This is because 
+    /// the runner will need to apply the runner's token to the request, and it is
+    /// a security best practice to *not* blindly add sensitive data to redirects
+    /// 302s.)
+    /// </summary>
+    [DataContract]
+    public class BrokerMigrationMessage
+    {
+        public static readonly string MessageType = "BrokerMigration";
+
+        public BrokerMigrationMessage()
+        {
+        }
+
+        public BrokerMigrationMessage(
+            Uri brokerUrl)
+        {
+            this.BrokerBaseUrl = brokerUrl;
+        }
+
+        /// <summary>
+        /// The base url for the broker listener
+        /// </summary>
+        [DataMember]
+        public Uri BrokerBaseUrl
+        {
+            get;
+            internal set;
+        }
+    }
+}

src/Sdk/DTWebApi/WebApi/Exceptions.cs

@@ -1539,6 +1539,26 @@ namespace GitHub.DistributedTask.WebApi
         }
     }
 
+    [Serializable]
+    [ExceptionMapping("0.0", "3.0", "TaskOrchestrationJobUnprocessableException", "GitHub.DistributedTask.WebApi.TaskOrchestrationJobUnprocessableException, GitHub.DistributedTask.WebApi, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a")]
+    public sealed class TaskOrchestrationJobUnprocessableException : DistributedTaskException
+    {
+        public TaskOrchestrationJobUnprocessableException(String message)
+            : base(message)
+        {
+        }
+
+        public TaskOrchestrationJobUnprocessableException(String message, Exception innerException)
+            : base(message, innerException)
+        {
+        }
+
+        private TaskOrchestrationJobUnprocessableException(SerializationInfo info, StreamingContext context)
+            : base(info, context)
+        {
+        }
+    }
+
     [Serializable]
     [ExceptionMapping("0.0", "3.0", "TaskOrchestrationPlanSecurityException", "GitHub.DistributedTask.WebApi.TaskOrchestrationPlanSecurityException, GitHub.DistributedTask.WebApi, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a")]
     public sealed class TaskOrchestrationPlanSecurityException : DistributedTaskException
@@ -2498,6 +2518,25 @@ namespace GitHub.DistributedTask.WebApi
         }
     }
 
+    [Serializable]
+    public class NonRetryableActionDownloadInfoException : DistributedTaskException
+    {
+        public NonRetryableActionDownloadInfoException(String message)
+            : base(message)
+        {
+        }
+
+        public NonRetryableActionDownloadInfoException(String message, Exception innerException)
+            : base(message, innerException)
+        {
+        }
+
+        protected NonRetryableActionDownloadInfoException(SerializationInfo info, StreamingContext context)
+            : base(info, context)
+        {
+        }
+    }
+
     [Serializable]
     public sealed class FailedToResolveActionDownloadInfoException : DistributedTaskException
     {

src/Sdk/DTWebApi/WebApi/TaskAgentHttpClient.cs

@@ -141,24 +141,6 @@ namespace GitHub.DistributedTask.WebApi
             return ReplaceAgentAsync(poolId, agent.Id, agent, userState, cancellationToken);
         }
 
-        public Task<Pipelines.AgentJobRequestMessage> GetJobMessageAsync(
-            string messageId,
-            object userState = null,
-            CancellationToken cancellationToken = default)
-        {
-            HttpMethod httpMethod = new HttpMethod("GET");
-            Guid locationId = new Guid("25adab70-1379-4186-be8e-b643061ebe3a");
-            object routeValues = new { messageId = messageId };
-
-            return SendAsync<Pipelines.AgentJobRequestMessage>(
-                httpMethod,
-                locationId,
-                routeValues: routeValues,
-                version: new ApiResourceVersion(6.0, 1),
-                userState: userState,
-                cancellationToken: cancellationToken);
-        }
-
         protected Task<T> SendAsync<T>(
             HttpMethod method,
             Guid locationId,

src/Sdk/DTWebApi/WebApi/TaskAgentMessageTypes.cs

@@ -0,0 +1,10 @@
+using System;
+using System.Runtime.Serialization;
+
+namespace GitHub.DistributedTask.WebApi
+{
+    public sealed class TaskAgentMessageTypes
+    {
+        public static readonly string ForceTokenRefresh = "ForceTokenRefresh";
+    }
+}

src/Sdk/DTWebApi/WebApi/TaskAgentSession.cs

@@ -75,5 +75,12 @@ namespace GitHub.DistributedTask.WebApi
             get;
             set;
         }
+
+        [DataMember(EmitDefaultValue = false, IsRequired = false)]
+        public BrokerMigrationMessage BrokerMigrationMessage
+        {
+            get;
+            set;
+        }
     }
 }

src/Sdk/DTWebApi/WebApi/TaskAttachment.cs

@@ -101,6 +101,7 @@ namespace GitHub.DistributedTask.WebApi
         public static readonly String FileAttachment = "DistributedTask.Core.FileAttachment";
         public static readonly String DiagnosticLog = "DistributedTask.Core.DiagnosticLog";
         public static readonly String ResultsLog = "Results.Core.Log";
+        public static readonly String ResultsDiagnosticLog = "Results.Core.DiagnosticLog";
     }
 
     [GenerateAllConstants]

src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs

@@ -7,5 +7,8 @@ namespace GitHub.Actions.RunService.WebApi
     {
         [DataMember(Name = "jobMessageId", EmitDefaultValue = false)]
         public string JobMessageId { get; set; }
+
+        [DataMember(Name = "runnerOS", EmitDefaultValue = false)]
+        public string RunnerOS { get; set; }
     }
 }

src/Sdk/RSWebApi/Contracts/BrokerError.cs

@@ -0,0 +1,20 @@
+using System.Runtime.Serialization;
+
+namespace GitHub.Actions.RunService.WebApi
+{
+    [DataContract]
+    public class BrokerError
+    {
+        [DataMember(Name = "source", EmitDefaultValue = false)]
+        public string Source { get; set; }
+
+        [DataMember(Name = "errorKind", EmitDefaultValue = false)]
+        public string ErrorKind { get; set; }
+
+        [DataMember(Name = "statusCode", EmitDefaultValue = false)]
+        public int StatusCode { get; set; }
+
+        [DataMember(Name = "errorMessage", EmitDefaultValue = false)]
+        public string Message { get; set; }
+    }
+}

src/Sdk/RSWebApi/Contracts/BrokerErrorKind.cs

@@ -0,0 +1,10 @@
+using System.Runtime.Serialization;
+
+namespace GitHub.Actions.RunService.WebApi
+{
+    [DataContract]
+    public class BrokerErrorKind
+    {
+        public const string RunnerVersionTooOld = "RunnerVersionTooOld";
+    }
+}

src/Sdk/RSWebApi/Contracts/RunServiceError.cs

@@ -0,0 +1,17 @@
+using System.Runtime.Serialization;
+
+namespace GitHub.Actions.RunService.WebApi
+{
+    [DataContract]
+    public class RunServiceError
+    {
+        [DataMember(Name = "source", EmitDefaultValue = false)]
+        public string Source { get; set; }
+
+        [DataMember(Name = "statusCode", EmitDefaultValue = false)]
+        public int Code { get; set; }
+
+        [DataMember(Name = "errorMessage", EmitDefaultValue = false)]
+        public string Message { get; set; }
+    }
+}

src/Sdk/RSWebApi/RunServiceHttpClient.cs

@@ -9,6 +9,7 @@ using GitHub.DistributedTask.WebApi;
 using GitHub.Services.Common;
 using GitHub.Services.OAuth;
 using GitHub.Services.WebApi;
+using Newtonsoft.Json;
 using Sdk.RSWebApi.Contracts;
 using Sdk.WebApi.WebApi;
 
@@ -16,6 +17,15 @@ namespace GitHub.Actions.RunService.WebApi
 {
     public class RunServiceHttpClient : RawHttpClientBase
     {
+        private static readonly JsonSerializerSettings s_serializerSettings;
+
+        static RunServiceHttpClient()
+        {
+            s_serializerSettings = new VssJsonMediaTypeFormatter().SerializerSettings;
+            s_serializerSettings.DateParseHandling = DateParseHandling.None;
+            s_serializerSettings.FloatParseHandling = FloatParseHandling.Double;
+        }
+
         public RunServiceHttpClient(
             Uri baseUrl,
             VssOAuthCredential credentials)
@@ -59,12 +69,14 @@ namespace GitHub.Actions.RunService.WebApi
         public async Task<AgentJobRequestMessage> GetJobMessageAsync(
             Uri requestUri,
             string messageId,
+            string runnerOS,
             CancellationToken cancellationToken = default)
         {
             HttpMethod httpMethod = new HttpMethod("POST");
             var payload = new AcquireJobRequest
             {
                 JobMessageId = messageId,
+                RunnerOS = runnerOS
             };
 
             requestUri = new Uri(requestUri, "acquirejob");
@@ -74,6 +86,7 @@ namespace GitHub.Actions.RunService.WebApi
                 httpMethod,
                 requestUri: requestUri,
                 content: requestContent,
+                readErrorBody: true,
                 cancellationToken: cancellationToken);
 
             if (result.IsSuccess)
@@ -81,14 +94,26 @@ namespace GitHub.Actions.RunService.WebApi
                 return result.Value;
             }
 
-            switch (result.StatusCode)
+            if (TryParseErrorBody(result.ErrorBody, out RunServiceError error))
             {
-                case HttpStatusCode.NotFound:
-                    throw new TaskOrchestrationJobNotFoundException($"Job message not found: {messageId}");
-                case HttpStatusCode.Conflict:
-                    throw new TaskOrchestrationJobAlreadyAcquiredException($"Job message already acquired: {messageId}");
-                default:
-                    throw new Exception($"Failed to get job message: {result.Error}");
+                switch ((HttpStatusCode)error.Code)
+                {
+                    case HttpStatusCode.NotFound:
+                        throw new TaskOrchestrationJobNotFoundException($"Job message not found '{messageId}'. {error.Message}");
+                    case HttpStatusCode.Conflict:
+                        throw new TaskOrchestrationJobAlreadyAcquiredException($"Job message already acquired '{messageId}'. {error.Message}");
+                    case HttpStatusCode.UnprocessableEntity:
+                        throw new TaskOrchestrationJobUnprocessableException($"Unprocessable job '{messageId}'. {error.Message}");
+                }
+            }
+
+            if (!string.IsNullOrEmpty(result.ErrorBody))
+            {
+                throw new Exception($"Failed to get job message: {result.Error}. {Truncate(result.ErrorBody)}");
+            }
+            else
+            {
+                throw new Exception($"Failed to get job message: {result.Error}");
             }
         }
 
@@ -96,7 +121,7 @@ namespace GitHub.Actions.RunService.WebApi
             Uri requestUri,
             Guid planId,
             Guid jobId,
-            TaskResult result,
+            TaskResult conclusion,
             Dictionary<String, VariableValue> outputs,
             IList<StepResult> stepResults,
             IList<Annotation> jobAnnotations,
@@ -108,7 +133,7 @@ namespace GitHub.Actions.RunService.WebApi
             {
                 PlanID = planId,
                 JobID = jobId,
-                Conclusion = result,
+                Conclusion = conclusion,
                 Outputs = outputs,
                 StepResults = stepResults,
                 Annotations = jobAnnotations,
@@ -118,22 +143,32 @@ namespace GitHub.Actions.RunService.WebApi
             requestUri = new Uri(requestUri, "completejob");
 
             var requestContent = new ObjectContent<CompleteJobRequest>(payload, new VssJsonMediaTypeFormatter(true));
-            var response = await SendAsync(
+            var result = await Send2Async(
                     httpMethod,
                     requestUri,
                     content: requestContent,
                     cancellationToken: cancellationToken);
-            if (response.IsSuccessStatusCode)
+            if (result.IsSuccess)
             {
                 return;
             }
 
-            switch (response.StatusCode)
+            if (TryParseErrorBody(result.ErrorBody, out RunServiceError error))
             {
-                case HttpStatusCode.NotFound:
-                    throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}");
-                default:
-                    throw new Exception($"Failed to complete job: {response.ReasonPhrase}");
+                switch ((HttpStatusCode)error.Code)
+                {
+                    case HttpStatusCode.NotFound:
+                        throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}. {error.Message}");
+                }
+            }
+
+            if (!string.IsNullOrEmpty(result.ErrorBody))
+            {
+                throw new Exception($"Failed to complete job: {result.Error}. {Truncate(result.ErrorBody)}");
+            }
+            else
+            {
+                throw new Exception($"Failed to complete job: {result.Error}");
             }
         }
 
@@ -157,6 +192,7 @@ namespace GitHub.Actions.RunService.WebApi
                 httpMethod,
                 requestUri,
                 content: requestContent,
+                readErrorBody: true,
                 cancellationToken: cancellationToken);
 
             if (result.IsSuccess)
@@ -164,13 +200,60 @@ namespace GitHub.Actions.RunService.WebApi
                 return result.Value;
             }
 
-            switch (result.StatusCode)
+            if (TryParseErrorBody(result.ErrorBody, out RunServiceError error))
             {
-                case HttpStatusCode.NotFound:
-                    throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}");
-                default:
-                    throw new Exception($"Failed to renew job: {result.Error}");
+                switch ((HttpStatusCode)error.Code)
+                {
+                    case HttpStatusCode.NotFound:
+                        throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}. {error.Message}");
+                }
             }
+
+            if (!string.IsNullOrEmpty(result.ErrorBody))
+            {
+                throw new Exception($"Failed to renew job: {result.Error}. {Truncate(result.ErrorBody)}");
+            }
+            else
+            {
+                throw new Exception($"Failed to renew job: {result.Error}");
+            }
+        }
+
+        protected override async Task<T> ReadJsonContentAsync<T>(HttpResponseMessage response, CancellationToken cancellationToken = default(CancellationToken))
+        {
+            var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
+            return JsonConvert.DeserializeObject<T>(json, s_serializerSettings);
+        }
+
+        private static bool TryParseErrorBody(string errorBody, out RunServiceError error)
+        {
+            if (!string.IsNullOrEmpty(errorBody))
+            {
+                try
+                {
+                    error = JsonUtility.FromString<RunServiceError>(errorBody);
+                    if (error?.Source == "actions-run-service")
+                    {
+                        return true;
+                    }
+                }
+                catch (Exception)
+                {
+                }
+            }
+
+            error = null;
+            return false;
+        }
+
+        private static string Truncate(string errorBody)
+        {
+            if (errorBody.Length > 100)
+            {
+                return errorBody.Substring(0, 100) + "[truncated]";
+            }
+
+            return errorBody;
         }
     }
 }

src/Sdk/Sdk.csproj

@@ -13,6 +13,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="Azure.Storage.Blobs" Version="12.19.1" />
         <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
         <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
         <PackageReference Include="Microsoft.AspNet.WebApi.Client" Version="5.2.9" />

src/Sdk/WebApi/WebApi/BrokerHttpClient.cs

@@ -57,6 +57,7 @@ namespace GitHub.Actions.RunService.WebApi
         }
 
         public async Task<TaskAgentMessage> GetRunnerMessageAsync(
+            Guid? sessionId,
             string runnerVersion,
             TaskAgentStatus? status,
             string os = null,
@@ -69,6 +70,11 @@ namespace GitHub.Actions.RunService.WebApi
 
             List<KeyValuePair<string, string>> queryParams = new List<KeyValuePair<string, string>>();
 
+            if (sessionId != null)
+            {
+                queryParams.Add("sessionId", sessionId.Value.ToString());
+            }
+
             if (status != null)
             {
                 queryParams.Add("status", status.Value.ToString());
@@ -97,6 +103,52 @@ namespace GitHub.Actions.RunService.WebApi
                 new HttpMethod("GET"),
                 requestUri: requestUri,
                 queryParameters: queryParams,
+                readErrorBody: true,
+                cancellationToken: cancellationToken);
+
+            if (result.IsSuccess)
+            {
+                return result.Value;
+            }
+
+            if (TryParseErrorBody(result.ErrorBody, out BrokerError brokerError))
+            {
+                switch (brokerError.ErrorKind)
+                {
+                    case BrokerErrorKind.RunnerVersionTooOld:
+                        throw new AccessDeniedException(brokerError.Message)
+                        {
+                            ErrorCode = 1
+                        };
+                    default:
+                        break;
+                }
+            }
+
+            // temporary back compat
+            if (result.StatusCode == HttpStatusCode.Forbidden)
+            {
+                throw new AccessDeniedException($"{result.Error} Runner version v{runnerVersion} is deprecated and cannot receive messages.")
+                {
+                    ErrorCode = 1
+                };
+            }
+
+            throw new Exception($"Failed to get job message. Request to {requestUri} failed with status: {result.StatusCode}. Error message {result.Error}");
+        }
+
+        public async Task<TaskAgentSession> CreateSessionAsync(
+
+           TaskAgentSession session,
+           CancellationToken cancellationToken = default)
+        {
+            var requestUri = new Uri(Client.BaseAddress, "session");
+            var requestContent = new ObjectContent<TaskAgentSession>(session, new VssJsonMediaTypeFormatter(true));
+
+            var result = await SendAsync<TaskAgentSession>(
+                new HttpMethod("POST"),
+                requestUri: requestUri,
+                content: requestContent,
                 cancellationToken: cancellationToken);
 
             if (result.IsSuccess)
@@ -109,7 +161,51 @@ namespace GitHub.Actions.RunService.WebApi
                 throw new AccessDeniedException(result.Error);
             }
 
-            throw new Exception($"Failed to get job message: {result.Error}");
+            if (result.StatusCode == HttpStatusCode.Conflict)
+            {
+                throw new TaskAgentSessionConflictException(result.Error);
+            }
+
+            throw new Exception($"Failed to create broker session: {result.Error}");
+        }
+
+        public async Task DeleteSessionAsync(
+            CancellationToken cancellationToken = default)
+        {
+            var requestUri = new Uri(Client.BaseAddress, $"session");
+
+            var result = await SendAsync<object>(
+                new HttpMethod("DELETE"),
+                requestUri: requestUri,
+                cancellationToken: cancellationToken);
+
+            if (result.IsSuccess)
+            {
+                return;
+            }
+
+            throw new Exception($"Failed to delete broker session: {result.Error}");
+        }
+
+        private static bool TryParseErrorBody(string errorBody, out BrokerError error)
+        {
+            if (!string.IsNullOrEmpty(errorBody))
+            {
+                try
+                {
+                    error = JsonUtility.FromString<BrokerError>(errorBody);
+                    if (error?.Source == "actions-broker-listener")
+                    {
+                        return true;
+                    }
+                }
+                catch (Exception)
+                {
+                }
+            }
+
+            error = null;
+            return false;
         }
     }
 }

src/Sdk/WebApi/WebApi/Contracts.cs

@@ -89,6 +89,26 @@ namespace GitHub.Services.Results.Contracts
         public long SoftSizeLimit;
     }
 
+    [DataContract]
+    [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
+    public class GetSignedDiagnosticLogsURLRequest
+    {
+        [DataMember]
+        public string WorkflowJobRunBackendId;
+        [DataMember]
+        public string WorkflowRunBackendId;
+    }
+
+    [DataContract]
+    [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
+    public class GetSignedDiagnosticLogsURLResponse
+    {
+        [DataMember]
+        public string DiagLogsURL;
+        [DataMember]
+        public string BlobStorageType;
+    }
+
     [DataContract]
     [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
     public class JobLogsMetadataCreate

src/Sdk/WebApi/WebApi/RawHttpClientBase.cs

@@ -101,7 +101,7 @@ namespace Sdk.WebApi.WebApi
             }
         }
 
-        protected Task<RawHttpClientResult<T>> SendAsync<T>(
+        protected async Task<RawHttpClientResult> Send2Async(
             HttpMethod method,
             Uri requestUri,
             HttpContent content = null,
@@ -109,7 +109,47 @@ namespace Sdk.WebApi.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
-            return SendAsync<T>(method, null, requestUri, content, queryParameters, userState, cancellationToken);
+            using (var response = await SendAsync(method, requestUri, content, queryParameters, userState, cancellationToken).ConfigureAwait(false))
+            {
+                if (response.IsSuccessStatusCode)
+                {
+                    return new RawHttpClientResult(
+                        isSuccess: true,
+                        error: string.Empty,
+                        statusCode: response.StatusCode);
+                }
+                else
+                {
+                    var errorBody = default(string);
+                    try
+                    {
+                        errorBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
+                    }
+                    catch (Exception ex)
+                    {
+                        errorBody = $"Error reading HTTP response body: {ex.Message}";
+                    }
+
+                    string errorMessage = $"Error: {response.ReasonPhrase}";
+                    return new RawHttpClientResult(
+                        isSuccess: false,
+                        error: errorMessage,
+                        statusCode: response.StatusCode,
+                        errorBody: errorBody);
+                }
+            }
+        }
+
+        protected Task<RawHttpClientResult<T>> SendAsync<T>(
+            HttpMethod method,
+            Uri requestUri,
+            HttpContent content = null,
+            IEnumerable<KeyValuePair<String, String>> queryParameters = null,
+            Boolean readErrorBody = false,
+            Object userState = null,
+            CancellationToken cancellationToken = default(CancellationToken))
+        {
+            return SendAsync<T>(method, null, requestUri, content, queryParameters, readErrorBody, userState, cancellationToken);
         }
 
         protected async Task<RawHttpClientResult<T>> SendAsync<T>(
@@ -118,18 +158,20 @@ namespace Sdk.WebApi.WebApi
             Uri requestUri,
             HttpContent content = null,
             IEnumerable<KeyValuePair<String, String>> queryParameters = null,
+            Boolean readErrorBody = false,
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
             using (VssTraceActivity.GetOrCreate().EnterCorrelationScope())
             using (HttpRequestMessage requestMessage = CreateRequestMessage(method, additionalHeaders, requestUri, content, queryParameters))
             {
-                return await SendAsync<T>(requestMessage, userState, cancellationToken).ConfigureAwait(false);
+                return await SendAsync<T>(requestMessage, readErrorBody, userState, cancellationToken).ConfigureAwait(false);
             }
         }
 
         protected async Task<RawHttpClientResult<T>> SendAsync<T>(
             HttpRequestMessage message,
+            Boolean readErrorBody = false,
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
@@ -145,8 +187,21 @@ namespace Sdk.WebApi.WebApi
                 }
                 else
                 {
+                    var errorBody = default(string);
+                    if (readErrorBody)
+                    {
+                        try
+                        {
+                            errorBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
+                        }
+                        catch (Exception ex)
+                        {
+                            errorBody = $"Error reading HTTP response body: {ex.Message}";
+                        }
+                    }
+
                     string errorMessage = $"Error: {response.ReasonPhrase}";
-                    return RawHttpClientResult<T>.Fail(errorMessage, response.StatusCode);
+                    return RawHttpClientResult<T>.Fail(errorMessage, response.StatusCode, errorBody);
                 }
             }
         }

src/Sdk/WebApi/WebApi/RawHttpClientResult.cs

@@ -5,15 +5,27 @@ namespace Sdk.WebApi.WebApi
     public class RawHttpClientResult
     {
         public bool IsSuccess { get; protected set; }
+
+        /// <summary>
+        /// A description of the HTTP status code, like "Error: Unprocessable Entity"
+        /// </summary>
         public string Error { get; protected set; }
+
+        /// <summary>
+        /// The HTTP response body for unsuccessful HTTP status codes, or an error message when reading the response body fails.
+        /// </summary>
+        public string ErrorBody { get; protected set; }
+
         public HttpStatusCode StatusCode { get; protected set; }
+
         public bool IsFailure => !IsSuccess;
 
-        protected RawHttpClientResult(bool isSuccess, string error, HttpStatusCode statusCode)
+        public RawHttpClientResult(bool isSuccess, string error, HttpStatusCode statusCode, string errorBody = null)
         {
             IsSuccess = isSuccess;
             Error = error;
             StatusCode = statusCode;
+            ErrorBody = errorBody;
         }
     }
 
@@ -21,13 +33,13 @@ namespace Sdk.WebApi.WebApi
     {
         public T Value { get; private set; }
 
-        protected internal RawHttpClientResult(T value, bool isSuccess, string error, HttpStatusCode statusCode)
-            : base(isSuccess, error, statusCode)
+        protected internal RawHttpClientResult(T value, bool isSuccess, string error, HttpStatusCode statusCode, string errorBody)
+            : base(isSuccess, error, statusCode, errorBody)
         {
             Value = value;
         }
 
-        public static RawHttpClientResult<T> Fail(string message, HttpStatusCode statusCode) => new RawHttpClientResult<T>(default(T), false, message, statusCode);
-        public static RawHttpClientResult<T> Ok(T value) => new RawHttpClientResult<T>(value, true, string.Empty, HttpStatusCode.OK);
+        public static RawHttpClientResult<T> Fail(string message, HttpStatusCode statusCode, string errorBody) => new RawHttpClientResult<T>(default(T), false, message, statusCode, errorBody);
+        public static RawHttpClientResult<T> Ok(T value) => new RawHttpClientResult<T>(value, true, string.Empty, HttpStatusCode.OK, null);
     }
 }

src/Sdk/WebApi/WebApi/ResultsHttpClient.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
@@ -8,8 +7,11 @@ using System.Net.Http.Headers;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Net.Http.Formatting;
+using Azure;
+using Azure.Storage.Blobs;
+using Azure.Storage.Blobs.Models;
+using Azure.Storage.Blobs.Specialized;
 using GitHub.DistributedTask.WebApi;
-using GitHub.Services.Common;
 using GitHub.Services.Results.Contracts;
 using Sdk.WebApi.WebApi;
 
@@ -21,13 +23,15 @@ namespace GitHub.Services.Results.Client
             Uri baseUrl,
             HttpMessageHandler pipeline,
             string token,
-            bool disposeHandler)
+            bool disposeHandler,
+            bool useSdk)
             : base(baseUrl, pipeline, disposeHandler)
         {
             m_token = token;
             m_resultsServiceUrl = baseUrl;
             m_formatter = new JsonMediaTypeFormatter();
             m_changeIdCounter = 1;
+            m_useSdk = useSdk;
         }
 
         // Get Sas URL calls
@@ -77,6 +81,19 @@ namespace GitHub.Services.Results.Client
             return await GetResultsSignedURLResponse<GetSignedStepLogsURLRequest, GetSignedStepLogsURLResponse>(getStepLogsSignedBlobURLEndpoint, cancellationToken, request);
         }
 
+        private async Task<GetSignedDiagnosticLogsURLResponse> GetDiagnosticLogsUploadUrlAsync(string planId, string jobId, CancellationToken cancellationToken)
+        {
+            var request = new GetSignedDiagnosticLogsURLRequest()
+            {
+                WorkflowJobRunBackendId = jobId,
+                WorkflowRunBackendId = planId,
+            };
+
+            var getDiagnosticLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetJobDiagLogsSignedBlobURL);
+
+            return await GetResultsSignedURLResponse<GetSignedDiagnosticLogsURLRequest, GetSignedDiagnosticLogsURLResponse>(getDiagnosticLogsSignedBlobURLEndpoint, cancellationToken, request);
+        }
+
         private async Task<GetSignedJobLogsURLResponse> GetJobLogUploadUrlAsync(string planId, string jobId, CancellationToken cancellationToken)
         {
             var request = new GetSignedJobLogsURLRequest()
@@ -91,7 +108,6 @@ namespace GitHub.Services.Results.Client
         }
 
         // Create metadata calls
-
         private async Task SendRequest<R>(Uri uri, CancellationToken cancellationToken, R request, string timestamp)
         {
             using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, uri))
@@ -161,73 +177,219 @@ namespace GitHub.Services.Results.Client
             await SendRequest<JobLogsMetadataCreate>(createJobLogsMetadataEndpoint, cancellationToken, request, timestamp);
         }
 
-        private async Task<HttpResponseMessage> UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken)
+        private (Uri path, string sas) ParseSasToken(string url)
         {
-            // Upload the file to the url
-            var request = new HttpRequestMessage(HttpMethod.Put, url)
+            if (String.IsNullOrEmpty(url))
             {
-                Content = new StreamContent(file)
-            };
-
-            if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
-            {
-                request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureBlockBlob);
+                throw new Exception($"SAS url is empty");
             }
 
-            using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
+            var blobUri = new UriBuilder(url);
+            var sasUrl = blobUri.Query.Substring(1); //remove starting "?"
+            blobUri.Query = null; // remove query params
+            return (blobUri.Uri, sasUrl);
+        }
+
+        private BlobClient GetBlobClient(string url)
+        {
+            var blobUri = ParseSasToken(url);
+
+            var opts = new BlobClientOptions
             {
-                if (!response.IsSuccessStatusCode)
+                Retry =
                 {
-                    throw new Exception($"Failed to upload file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}");
+                    MaxRetries = Constants.DefaultBlobUploadRetries,
+                    NetworkTimeout = TimeSpan.FromSeconds(Constants.DefaultNetworkTimeoutInSeconds)
+                }
+            };
+
+            return new BlobClient(blobUri.path, new AzureSasCredential(blobUri.sas), opts);
+        }
+
+        private AppendBlobClient GetAppendBlobClient(string url)
+        {
+            var blobUri = ParseSasToken(url);
+
+            var opts = new BlobClientOptions
+            {
+                Retry =
+                {
+                    MaxRetries = Constants.DefaultBlobUploadRetries,
+                    NetworkTimeout = TimeSpan.FromSeconds(Constants.DefaultNetworkTimeoutInSeconds)
+                }
+            };
+
+            return new AppendBlobClient(blobUri.path, new AzureSasCredential(blobUri.sas), opts);
+        }
+
+        private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken, Dictionary<string, string> customHeaders = null)
+        {
+            if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage)
+            {
+                var blobClient = GetBlobClient(url);
+                var httpHeaders = new BlobHttpHeaders();
+                if (customHeaders != null)
+                {
+                    foreach (var header in customHeaders)
+                    {
+                        switch (header.Key)
+                        {
+                            case Constants.ContentTypeHeader:
+                                httpHeaders.ContentType = header.Value;
+                                break;
+                        }
+                    }
+                }
+                try
+                {
+                    await blobClient.UploadAsync(file, new BlobUploadOptions()
+                    {
+                        HttpHeaders = httpHeaders,
+                        Conditions = new BlobRequestConditions
+                        {
+                            IfNoneMatch = new ETag("*")
+                        }
+                    }, cancellationToken);
+                }
+                catch (RequestFailedException e)
+                {
+                    throw new Exception($"Failed to upload block to Azure blob: {e.Message}");
+                }
+            }
+            else
+            {
+                // Upload the file to the url
+                var request = new HttpRequestMessage(HttpMethod.Put, url)
+                {
+                    Content = new StreamContent(file)
+                };
+
+                if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
+                {
+                    request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureBlockBlob);
+                }
+
+                if (customHeaders != null)
+                {
+                    foreach (var header in customHeaders)
+                    {
+                        request.Content.Headers.Add(header.Key, header.Value);
+                    }
+                };
+
+                using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
+                {
+                    if (!response.IsSuccessStatusCode)
+                    {
+                        throw new Exception($"Failed to upload file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}");
+                    }
                 }
-                return response;
             }
         }
 
-        private async Task<HttpResponseMessage> CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken)
+        private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken, Dictionary<string, string> customHeaders = null)
         {
-            var request = new HttpRequestMessage(HttpMethod.Put, url)
+            if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage)
             {
-                Content = new StringContent("")
-            };
-            if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
-            {
-                request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureAppendBlob);
-                request.Content.Headers.Add("Content-Length", "0");
-            }
-
-            using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
-            {
-                if (!response.IsSuccessStatusCode)
+                var appendBlobClient = GetAppendBlobClient(url);
+                var httpHeaders = new BlobHttpHeaders();
+                if (customHeaders != null)
                 {
-                    throw new Exception($"Failed to create append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}");
+                    foreach (var header in customHeaders)
+                    {
+                        switch (header.Key)
+                        {
+                            case Constants.ContentTypeHeader:
+                                httpHeaders.ContentType = header.Value;
+                                break;
+                        }
+                    }
+                }
+                try
+                {
+                    await appendBlobClient.CreateAsync(new AppendBlobCreateOptions()
+                    {
+                        HttpHeaders = httpHeaders,
+                        Conditions = new AppendBlobRequestConditions
+                        {
+                            IfNoneMatch = new ETag("*")
+                        }
+                    }, cancellationToken: cancellationToken);
+                }
+                catch (RequestFailedException e)
+                {
+                    throw new Exception($"Failed to create append blob in Azure blob: {e.Message}");
+                }
+            }
+            else
+            {
+                var request = new HttpRequestMessage(HttpMethod.Put, url)
+                {
+                    Content = new StringContent("")
+                };
+                if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
+                {
+                    request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureAppendBlob);
+                    request.Content.Headers.Add("Content-Length", "0");
+                }
+                if (customHeaders != null)
+                {
+                    foreach (var header in customHeaders)
+                    {
+                        request.Content.Headers.Add(header.Key, header.Value);
+                    }
+                };
+
+                using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
+                {
+                    if (!response.IsSuccessStatusCode)
+                    {
+                        throw new Exception($"Failed to create append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}");
+                    }
                 }
-                return response;
             }
         }
 
-        private async Task<HttpResponseMessage> UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken)
+        private async Task UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken)
         {
-            var comp = finalize ? "&comp=appendblock&seal=true" : "&comp=appendblock";
-            // Upload the file to the url
-            var request = new HttpRequestMessage(HttpMethod.Put, url + comp)
+            if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage)
             {
-                Content = new StreamContent(file)
-            };
-
-            if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
-            {
-                request.Content.Headers.Add("Content-Length", fileSize.ToString());
-                request.Content.Headers.Add(Constants.AzureBlobSealedHeader, finalize.ToString());
-            }
-
-            using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
-            {
-                if (!response.IsSuccessStatusCode)
+                var appendBlobClient = GetAppendBlobClient(url);
+                try
                 {
-                    throw new Exception($"Failed to upload append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}, object: {response}, fileSize: {fileSize}");
+                    await appendBlobClient.AppendBlockAsync(file, cancellationToken: cancellationToken);
+                    if (finalize)
+                    {
+                        await appendBlobClient.SealAsync(cancellationToken: cancellationToken);
+                    }
+                }
+                catch (RequestFailedException e)
+                {
+                    throw new Exception($"Failed to upload append block in Azure blob: {e.Message}");
+                }
+            }
+            else
+            {
+                var comp = finalize ? "&comp=appendblock&seal=true" : "&comp=appendblock";
+                // Upload the file to the url
+                var request = new HttpRequestMessage(HttpMethod.Put, url + comp)
+                {
+                    Content = new StreamContent(file)
+                };
+
+                if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
+                {
+                    request.Content.Headers.Add("Content-Length", fileSize.ToString());
+                    request.Content.Headers.Add(Constants.AzureBlobSealedHeader, finalize.ToString());
+                }
+
+                using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
+                {
+                    if (!response.IsSuccessStatusCode)
+                    {
+                        throw new Exception($"Failed to upload append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}, object: {response}, fileSize: {fileSize}");
+                    }
                 }
-                return response;
             }
         }
 
@@ -251,23 +413,22 @@ namespace GitHub.Services.Results.Client
             // Upload the file
             using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
             {
-                var response = await UploadBlockFileAsync(uploadUrlResponse.SummaryUrl, uploadUrlResponse.BlobStorageType, fileStream, cancellationToken);
+                await UploadBlockFileAsync(uploadUrlResponse.SummaryUrl, uploadUrlResponse.BlobStorageType, fileStream, cancellationToken);
             }
 
             // Send step summary upload complete message
             await StepSummaryUploadCompleteAsync(planId, jobId, stepId, fileSize, cancellationToken);
         }
 
-        private async Task<HttpResponseMessage> UploadLogFile(string file, bool finalize, bool firstBlock, string sasUrl, string blobStorageType,
-            CancellationToken cancellationToken)
+        private async Task UploadLogFile(string file, bool finalize, bool firstBlock, string sasUrl, string blobStorageType,
+            CancellationToken cancellationToken, Dictionary<string, string> customHeaders = null)
         {
-            HttpResponseMessage response;
             if (firstBlock && finalize)
             {
                 // This is the one and only block, just use a block blob
                 using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
                 {
-                    response = await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken);
+                    await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken, customHeaders);
                 }
             }
             else
@@ -276,18 +437,16 @@ namespace GitHub.Services.Results.Client
                 // Create the Append blob
                 if (firstBlock)
                 {
-                    await CreateAppendFileAsync(sasUrl, blobStorageType, cancellationToken);
+                    await CreateAppendFileAsync(sasUrl, blobStorageType, cancellationToken, customHeaders);
                 }
 
                 // Upload content
                 var fileSize = new FileInfo(file).Length;
                 using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
                 {
-                    response = await UploadAppendFileAsync(sasUrl, blobStorageType, fileStream, finalize, fileSize, cancellationToken);
+                    await UploadAppendFileAsync(sasUrl, blobStorageType, fileStream, finalize, fileSize, cancellationToken);
                 }
             }
-
-            return response;
         }
 
         // Handle file upload for step log
@@ -300,7 +459,12 @@ namespace GitHub.Services.Results.Client
                 throw new Exception("Failed to get step log upload url");
             }
 
-            await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken);
+            var customHeaders = new Dictionary<string, string>
+            {
+                { Constants.ContentTypeHeader, Constants.TextPlainContentType }
+            };
+
+            await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken, customHeaders);
 
             // Update metadata
             if (finalize)
@@ -320,7 +484,12 @@ namespace GitHub.Services.Results.Client
                 throw new Exception("Failed to get job log upload url");
             }
 
-            await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken);
+            var customHeaders = new Dictionary<string, string>
+            {
+                { Constants.ContentTypeHeader, Constants.TextPlainContentType }
+            };
+
+            await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken, customHeaders);
 
             // Update metadata
             if (finalize)
@@ -330,6 +499,18 @@ namespace GitHub.Services.Results.Client
             }
         }
 
+        public async Task UploadResultsDiagnosticLogsAsync(string planId, string jobId, string file, CancellationToken cancellationToken)
+        {
+            // Get the upload url
+            var uploadUrlResponse = await GetDiagnosticLogsUploadUrlAsync(planId, jobId, cancellationToken);
+            if (uploadUrlResponse == null || uploadUrlResponse.DiagLogsURL == null)
+            {
+                throw new Exception("Failed to get diagnostic logs upload url");
+            }
+
+            await UploadLogFile(file, true, true, uploadUrlResponse.DiagLogsURL, uploadUrlResponse.BlobStorageType, cancellationToken);
+        }
+
         private Step ConvertTimelineRecordToStep(TimelineRecord r)
         {
             return new Step()
@@ -405,6 +586,7 @@ namespace GitHub.Services.Results.Client
         private Uri m_resultsServiceUrl;
         private string m_token;
         private int m_changeIdCounter;
+        private bool m_useSdk;
     }
 
     // Constants specific to results
@@ -419,13 +601,20 @@ namespace GitHub.Services.Results.Client
         public static readonly string CreateStepLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateStepLogsMetadata";
         public static readonly string GetJobLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetJobLogsSignedBlobURL";
         public static readonly string CreateJobLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateJobLogsMetadata";
+        public static readonly string GetJobDiagLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetJobDiagLogsSignedBlobURL";
         public static readonly string ResultsProtoApiV1Endpoint = "twirp/github.actions.results.api.v1.WorkflowStepUpdateService/";
         public static readonly string WorkflowStepsUpdate = ResultsProtoApiV1Endpoint + "WorkflowStepsUpdate";
 
+        public static readonly int DefaultNetworkTimeoutInSeconds = 30;
+        public static readonly int DefaultBlobUploadRetries = 3;
+
         public static readonly string AzureBlobSealedHeader = "x-ms-blob-sealed";
         public static readonly string AzureBlobTypeHeader = "x-ms-blob-type";
         public static readonly string AzureBlockBlob = "BlockBlob";
         public static readonly string AzureAppendBlob = "AppendBlob";
+
+        public const string ContentTypeHeader = "Content-Type";
+        public const string TextPlainContentType = "text/plain";
     }
 
 }

src/Test/L0/CommandLineParserL0.cs

@@ -68,7 +68,7 @@ namespace GitHub.Runner.Common.Tests
                 trace.Info("Parsed");
 
                 trace.Info("Commands: {0}", clp.Commands.Count);
-                Assert.True(clp.Commands.Count == 2);
+                Assert.Equal(2, clp.Commands.Count);
             }
         }
 
@@ -88,7 +88,7 @@ namespace GitHub.Runner.Common.Tests
                 trace.Info("Parsed");
 
                 trace.Info("Args: {0}", clp.Args.Count);
-                Assert.True(clp.Args.Count == 2);
+                Assert.Equal(2, clp.Args.Count);
                 Assert.True(clp.Args.ContainsKey("arg1"));
                 Assert.Equal("arg1val", clp.Args["arg1"]);
                 Assert.True(clp.Args.ContainsKey("arg2"));
@@ -112,7 +112,7 @@ namespace GitHub.Runner.Common.Tests
                 trace.Info("Parsed");
 
                 trace.Info("Args: {0}", clp.Flags.Count);
-                Assert.True(clp.Flags.Count == 2);
+                Assert.Equal(2, clp.Flags.Count);
                 Assert.Contains("flag1", clp.Flags);
                 Assert.Contains("flag2", clp.Flags);
             }

src/Test/L0/ConstantGenerationL0.cs

@@ -24,7 +24,7 @@ namespace GitHub.Runner.Common.Tests
                 "osx-arm64"
             };
 
-            Assert.True(BuildConstants.Source.CommitHash.Length == 40, $"CommitHash should be SHA-1 hash {BuildConstants.Source.CommitHash}");
+            Assert.Equal(40, BuildConstants.Source.CommitHash.Length);
             Assert.True(validPackageNames.Contains(BuildConstants.RunnerPackage.PackageName), $"PackageName should be one of the following '{string.Join(", ", validPackageNames)}', current PackageName is '{BuildConstants.RunnerPackage.PackageName}'");
         }
     }

src/Test/L0/Listener/BrokerMessageListenerL0.cs

@@ -0,0 +1,81 @@
+using System;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Listener;
+using GitHub.Runner.Listener.Configuration;
+using GitHub.Services.Common;
+using Moq;
+using Xunit;
+
+namespace GitHub.Runner.Common.Tests.Listener
+{
+    public sealed class BrokerMessageListenerL0
+    {
+        private readonly RunnerSettings _settings;
+        private readonly Mock<IConfigurationManager> _config;
+        private readonly Mock<IBrokerServer> _brokerServer;
+        private readonly Mock<ICredentialManager> _credMgr;
+        private Mock<IConfigurationStore> _store;
+
+
+        public BrokerMessageListenerL0()
+        {
+            _settings = new RunnerSettings { AgentId = 1, AgentName = "myagent", PoolId = 123, PoolName = "default", ServerUrl = "http://myserver", WorkFolder = "_work", ServerUrlV2 = "http://myserverv2" };
+            _config = new Mock<IConfigurationManager>();
+            _config.Setup(x => x.LoadSettings()).Returns(_settings);
+            _credMgr = new Mock<ICredentialManager>();
+            _store = new Mock<IConfigurationStore>();
+            _brokerServer = new Mock<IBrokerServer>();
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void CreatesSession()
+        {
+            using (TestHostContext tc = CreateTestContext())
+            using (var tokenSource = new CancellationTokenSource())
+            {
+                Tracing trace = tc.GetTrace();
+
+                // Arrange.
+                var expectedSession = new TaskAgentSession();
+                _brokerServer
+                    .Setup(x => x.CreateSessionAsync(
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedSession));
+
+                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
+                _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
+
+                // Act.
+                BrokerMessageListener listener = new();
+                listener.Initialize(tc);
+
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
+                trace.Info("result: {0}", result);
+
+                // Assert.
+                Assert.Equal(CreateSessionResult.Success, result);
+                _brokerServer
+                   .Verify(x => x.CreateSessionAsync(
+                       It.Is<TaskAgentSession>(y => y != null),
+                       tokenSource.Token), Times.Once());
+            }
+        }
+
+        private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
+        {
+            TestHostContext tc = new(this, testName);
+            tc.SetSingleton<IConfigurationManager>(_config.Object);
+            tc.SetSingleton<ICredentialManager>(_credMgr.Object);
+            tc.SetSingleton<IConfigurationStore>(_store.Object);
+            tc.SetSingleton<IBrokerServer>(_brokerServer.Object);
+            return tc;
+        }
+    }
+}