[Linux,macOS,Windows] Download two latest CodeQL bundles (#6884)

Previously, our runner image only included the most recent release of CodeQL. In order to improve stability of rollout of new CodeQL versions, we are now downloading the most recent two releases.

The change updates the Linux and Windows script variable names to match that of the Mac script, because they are mostly functionally identical. It also adds the same log messages to all scripts and ensures Mac and Windows test parity. It tests that there are two bundles in each of the OS's as well.

This will approximately double the total size and installation time of the existing CodeQL tool.

images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1

@@ -103,7 +103,7 @@ $tools.AddToolVersion("Bazelisk", $(Get-BazeliskVersion))
 $tools.AddToolVersion("Bicep", $(Get-BicepVersion))
 $tools.AddToolVersion("Buildah", $(Get-BuildahVersion))
 $tools.AddToolVersion("CMake", $(Get-CMakeVersion))
-$tools.AddToolVersion("CodeQL Action Bundle", $(Get-CodeQLBundleVersion))
+$tools.AddToolVersion("CodeQL Action Bundles", $(Get-CodeQLBundleVersions))
 $tools.AddToolVersion("Docker Amazon ECR Credential Helper", $(Get-DockerAmazonECRCredHelperVersion))
 $tools.AddToolVersion("Docker Compose v1", $(Get-DockerComposeV1Version))
 $tools.AddToolVersion("Docker Compose v2", $(Get-DockerComposeV2Version))

images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1

@@ -30,12 +30,17 @@ function Get-BicepVersion {
     return $Matches.Version
 }
 
-function Get-CodeQLBundleVersion {
+function Get-CodeQLBundleVersions {
     $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*"
-    $CodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Select-Object -First 1 -Expand FullName
-    $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql"
-    $CodeQLVersion = & $CodeQLPath version --quiet
-    return $CodeQLVersion
+    $CodeQLVersionPaths = Get-ChildItem $CodeQLVersionsWildcard 
+    $CodeQlVersions=@()
+    foreach ($CodeQLVersionPath in $CodeQLVersionPaths) {
+        $FullCodeQLVersionPath = $CodeQLVersionPath | Select-Object -Expand FullName
+        $CodeQLPath = Join-Path $FullCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql"
+        $CodeQLVersion = & $CodeQLPath version --quiet
+        $CodeQLVersions += $CodeQLVersion
+    }
+    return $CodeQLVersions
 }
 
 function Get-PodManVersion {

images/linux/scripts/installers/codeql-bundle.sh

@@ -7,19 +7,39 @@
 source $HELPER_SCRIPTS/install.sh
 
 # Retrieve the name of the CodeQL bundle preferred by the Action (in the format codeql-bundle-YYYYMMDD).
-codeql_bundle_name="$(curl -sSL https://raw.githubusercontent.com/github/codeql-action/v2/src/defaults.json | jq -r .bundleVersion)"
-# Convert the bundle name to a version number (0.0.0-YYYYMMDD).
-codeql_bundle_version="0.0.0-${codeql_bundle_name##*-}"
+base_url="$(curl -sSL https://raw.githubusercontent.com/github/codeql-action/v2/src/defaults.json)"
+codeql_tag_name="$(echo "$base_url" | jq -r '.bundleVersion')"
+codeql_cli_version="$(echo "$base_url" | jq -r '.cliVersion')"
+prior_codeql_tag_name="$(echo "$base_url" | jq -r '.priorBundleVersion')"
+prior_codeql_cli_version="$(echo "$base_url" | jq -r '.priorCliVersion')"
 
-extraction_directory="$AGENT_TOOLSDIRECTORY/CodeQL/$codeql_bundle_version/x64"
-mkdir -p "$extraction_directory"
+# Convert the tag names to bundles with a version number (x.y.z-YYYYMMDD).
+codeql_bundle_version="${codeql_cli_version}-${codeql_tag_name##*-}"
+prior_codeql_bundle_version="${prior_codeql_cli_version}-${prior_codeql_tag_name##*-}"
 
-echo "Downloading CodeQL bundle $codeql_bundle_version..."
-download_with_retries "https://github.com/github/codeql-action/releases/download/$codeql_bundle_name/codeql-bundle.tar.gz" "/tmp" "codeql-bundle.tar.gz"
-tar -xzf "/tmp/codeql-bundle.tar.gz" -C "$extraction_directory"
+# Download and name both CodeQL bundles.
+codeql_bundle_versions=("${codeql_bundle_version}" "${prior_codeql_bundle_version}")
+codeql_tag_names=("${codeql_tag_name}" "${prior_codeql_tag_name}")
 
-# Touch a special file that indicates to the CodeQL Action that this bundle was baked-in to the hosted runner images.
-touch "$extraction_directory/pinned-version"
+for index in "${!codeql_bundle_versions[@]}"; do
+  bundle_version="${codeql_bundle_versions[$index]}"
+  bundle_tag_name="${codeql_tag_names[$index]}"
+  
+  echo "Downloading CodeQL bundle $bundle_version..."
+    download_with_retries "https://github.com/github/codeql-action/releases/download/$bundle_tag_name/codeql-bundle.tar.gz" "/tmp" "codeql-bundle.tar.gz"
+    codeql_archive="/tmp/codeql-bundle.tar.gz"
 
-# Touch a file to indicate to the toolcache that setting up CodeQL is complete.
-touch "$extraction_directory.complete"
+    codeql_toolcache_path="$AGENT_TOOLSDIRECTORY/CodeQL/$bundle_version/x64"
+    mkdir -p "$codeql_toolcache_path"
+
+    echo "Unpacking the downloaded CodeQL bundle archive..."
+    tar -xzf "$codeql_archive" -C "$codeql_toolcache_path"
+
+    # We only pin the latest version in the toolcache, to support overriding the CodeQL version specified in defaults.json on GitHub Enterprise.
+    if [[ "$bundle_version" == "$codeql_bundle_version" ]]; then
+        touch "$codeql_toolcache_path/pinned-version"
+    fi
+
+    # Touch a file to indicate to the toolcache that setting up CodeQL is complete.
+    touch "$codeql_toolcache_path.complete"
+done

images/linux/scripts/tests/Toolset.Tests.ps1

@@ -60,6 +60,19 @@ Describe "Toolset" {
                         }
                     }
                 }
+
+                # Validate that there are two versions of CodeQL included. 
+                if ($toolName -eq "CodeQL") {
+                    $foundPriorVersion = Get-Item $expectedVersionPath `
+                        | Sort-Object -Property {[SemVer]$_.name} -Descending `
+                        | Select-Object -Last 1
+                    $foundPriorVersionPath = Join-Path $foundPriorVersion $tool.arch
+
+                    $priorExecutablePath = Join-Path $foundPriorVersionPath "codeql/codeql"   
+                    It "Validate prior version of codeql/codeql" -TestCases @{PriorExecutablePath = $priorExecutablePath} {
+                        $PriorExecutablePath | Should -Exist
+                    }
+                }
             }
         }
     }