From 1e75a20a49556ec94abfcd8ff5f2bff26de5ed43 Mon Sep 17 00:00:00 2001 From: Angela P Wen Date: Sun, 29 Jan 2023 05:46:43 -0800 Subject: [PATCH] [Linux,macOS,Windows] Download two latest CodeQL bundles (#6884) Previously, our runner image only included the most recent release of CodeQL. In order to improve stability of rollout of new CodeQL versions, we are now downloading the most recent two releases. The change updates the Linux and Windows script variable names to match that of the Mac script, because they are mostly functionally identical. It also adds the same log messages to all scripts and ensures Mac and Windows test parity. It tests that there are two bundles in each of the OS's as well. This will approximately double the total size and installation time of the existing CodeQL tool. --- .../SoftwareReport.Generator.ps1 | 2 +- .../SoftwareReport/SoftwareReport.Tools.psm1 | 15 +++-- .../linux/scripts/installers/codeql-bundle.sh | 44 ++++++++++---- images/linux/scripts/tests/Toolset.Tests.ps1 | 13 +++++ images/macos/provision/core/codeql-bundle.sh | 47 ++++++++++----- .../SoftwareReport.Common.psm1 | 17 ++++-- .../SoftwareReport.Generator.ps1 | 2 +- images/macos/tests/Common.Tests.ps1 | 40 ++++++++++--- .../Installers/Install-CodeQLBundle.ps1 | 57 +++++++++++++------ .../SoftwareReport.Generator.ps1 | 2 +- .../SoftwareReport/SoftwareReport.Tools.psm1 | 17 ++++-- images/win/scripts/Tests/Tools.Tests.ps1 | 40 +++++++++++-- 12 files changed, 218 insertions(+), 78 deletions(-) diff --git a/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 b/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 index cb0f26b5e..ac7c8daa6 100644 --- a/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 +++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 @@ -103,7 +103,7 @@ $tools.AddToolVersion("Bazelisk", $(Get-BazeliskVersion)) $tools.AddToolVersion("Bicep", $(Get-BicepVersion)) $tools.AddToolVersion("Buildah", $(Get-BuildahVersion)) $tools.AddToolVersion("CMake", $(Get-CMakeVersion)) -$tools.AddToolVersion("CodeQL Action Bundle", $(Get-CodeQLBundleVersion)) +$tools.AddToolVersion("CodeQL Action Bundles", $(Get-CodeQLBundleVersions)) $tools.AddToolVersion("Docker Amazon ECR Credential Helper", $(Get-DockerAmazonECRCredHelperVersion)) $tools.AddToolVersion("Docker Compose v1", $(Get-DockerComposeV1Version)) $tools.AddToolVersion("Docker Compose v2", $(Get-DockerComposeV2Version)) diff --git a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 index c18f2e814..5dc675471 100644 --- a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 +++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 @@ -30,12 +30,17 @@ function Get-BicepVersion { return $Matches.Version } -function Get-CodeQLBundleVersion { +function Get-CodeQLBundleVersions { $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" - $CodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Select-Object -First 1 -Expand FullName - $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" - $CodeQLVersion = & $CodeQLPath version --quiet - return $CodeQLVersion + $CodeQLVersionPaths = Get-ChildItem $CodeQLVersionsWildcard + $CodeQlVersions=@() + foreach ($CodeQLVersionPath in $CodeQLVersionPaths) { + $FullCodeQLVersionPath = $CodeQLVersionPath | Select-Object -Expand FullName + $CodeQLPath = Join-Path $FullCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" + $CodeQLVersion = & $CodeQLPath version --quiet + $CodeQLVersions += $CodeQLVersion + } + return $CodeQLVersions } function Get-PodManVersion { diff --git a/images/linux/scripts/installers/codeql-bundle.sh b/images/linux/scripts/installers/codeql-bundle.sh index f2167b8c4..322f474b5 100644 --- a/images/linux/scripts/installers/codeql-bundle.sh +++ b/images/linux/scripts/installers/codeql-bundle.sh @@ -7,19 +7,39 @@ source $HELPER_SCRIPTS/install.sh # Retrieve the name of the CodeQL bundle preferred by the Action (in the format codeql-bundle-YYYYMMDD). -codeql_bundle_name="$(curl -sSL https://raw.githubusercontent.com/github/codeql-action/v2/src/defaults.json | jq -r .bundleVersion)" -# Convert the bundle name to a version number (0.0.0-YYYYMMDD). -codeql_bundle_version="0.0.0-${codeql_bundle_name##*-}" +base_url="$(curl -sSL https://raw.githubusercontent.com/github/codeql-action/v2/src/defaults.json)" +codeql_tag_name="$(echo "$base_url" | jq -r '.bundleVersion')" +codeql_cli_version="$(echo "$base_url" | jq -r '.cliVersion')" +prior_codeql_tag_name="$(echo "$base_url" | jq -r '.priorBundleVersion')" +prior_codeql_cli_version="$(echo "$base_url" | jq -r '.priorCliVersion')" -extraction_directory="$AGENT_TOOLSDIRECTORY/CodeQL/$codeql_bundle_version/x64" -mkdir -p "$extraction_directory" +# Convert the tag names to bundles with a version number (x.y.z-YYYYMMDD). +codeql_bundle_version="${codeql_cli_version}-${codeql_tag_name##*-}" +prior_codeql_bundle_version="${prior_codeql_cli_version}-${prior_codeql_tag_name##*-}" -echo "Downloading CodeQL bundle $codeql_bundle_version..." -download_with_retries "https://github.com/github/codeql-action/releases/download/$codeql_bundle_name/codeql-bundle.tar.gz" "/tmp" "codeql-bundle.tar.gz" -tar -xzf "/tmp/codeql-bundle.tar.gz" -C "$extraction_directory" +# Download and name both CodeQL bundles. +codeql_bundle_versions=("${codeql_bundle_version}" "${prior_codeql_bundle_version}") +codeql_tag_names=("${codeql_tag_name}" "${prior_codeql_tag_name}") -# Touch a special file that indicates to the CodeQL Action that this bundle was baked-in to the hosted runner images. -touch "$extraction_directory/pinned-version" +for index in "${!codeql_bundle_versions[@]}"; do + bundle_version="${codeql_bundle_versions[$index]}" + bundle_tag_name="${codeql_tag_names[$index]}" + + echo "Downloading CodeQL bundle $bundle_version..." + download_with_retries "https://github.com/github/codeql-action/releases/download/$bundle_tag_name/codeql-bundle.tar.gz" "/tmp" "codeql-bundle.tar.gz" + codeql_archive="/tmp/codeql-bundle.tar.gz" -# Touch a file to indicate to the toolcache that setting up CodeQL is complete. -touch "$extraction_directory.complete" + codeql_toolcache_path="$AGENT_TOOLSDIRECTORY/CodeQL/$bundle_version/x64" + mkdir -p "$codeql_toolcache_path" + + echo "Unpacking the downloaded CodeQL bundle archive..." + tar -xzf "$codeql_archive" -C "$codeql_toolcache_path" + + # We only pin the latest version in the toolcache, to support overriding the CodeQL version specified in defaults.json on GitHub Enterprise. + if [[ "$bundle_version" == "$codeql_bundle_version" ]]; then + touch "$codeql_toolcache_path/pinned-version" + fi + + # Touch a file to indicate to the toolcache that setting up CodeQL is complete. + touch "$codeql_toolcache_path.complete" +done diff --git a/images/linux/scripts/tests/Toolset.Tests.ps1 b/images/linux/scripts/tests/Toolset.Tests.ps1 index ef331c0b6..981d6093d 100644 --- a/images/linux/scripts/tests/Toolset.Tests.ps1 +++ b/images/linux/scripts/tests/Toolset.Tests.ps1 @@ -60,6 +60,19 @@ Describe "Toolset" { } } } + + # Validate that there are two versions of CodeQL included. + if ($toolName -eq "CodeQL") { + $foundPriorVersion = Get-Item $expectedVersionPath ` + | Sort-Object -Property {[SemVer]$_.name} -Descending ` + | Select-Object -Last 1 + $foundPriorVersionPath = Join-Path $foundPriorVersion $tool.arch + + $priorExecutablePath = Join-Path $foundPriorVersionPath "codeql/codeql" + It "Validate prior version of codeql/codeql" -TestCases @{PriorExecutablePath = $priorExecutablePath} { + $PriorExecutablePath | Should -Exist + } + } } } } diff --git a/images/macos/provision/core/codeql-bundle.sh b/images/macos/provision/core/codeql-bundle.sh index 1677363cf..53210a3d9 100644 --- a/images/macos/provision/core/codeql-bundle.sh +++ b/images/macos/provision/core/codeql-bundle.sh @@ -2,24 +2,41 @@ source ~/utils/utils.sh # Retrieve the name of the CodeQL bundle preferred by the Action (in the format codeql-bundle-YYYYMMDD). -codeql_bundle_name="$(curl -sSL https://raw.githubusercontent.com/github/codeql-action/v2/src/defaults.json | jq -r .bundleVersion)" -# Convert the bundle name to a version number (0.0.0-YYYYMMDD). -codeql_bundle_version="0.0.0-${codeql_bundle_name##*-}" +base_url="$(curl -sSL https://raw.githubusercontent.com/github/codeql-action/v2/src/defaults.json)" +codeql_tag_name="$(echo "$base_url" | jq -r '.bundleVersion')" +codeql_cli_version="$(echo "$base_url" | jq -r '.cliVersion')" +prior_codeql_tag_name="$(echo "$base_url" | jq -r '.priorBundleVersion')" +prior_codeql_cli_version="$(echo "$base_url" | jq -r '.priorCliVersion')" -echo "Downloading CodeQL bundle $codeql_bundle_version..." -download_with_retries "https://github.com/github/codeql-action/releases/download/$codeql_bundle_name/codeql-bundle.tar.gz" "/tmp" "codeql-bundle.tar.gz" -codeqlArchive="/tmp/codeql-bundle.tar.gz" +# Convert the tag names to bundles with a version number (x.y.z-YYYYMMDD). +codeql_bundle_version="${codeql_cli_version}-${codeql_tag_name##*-}" +prior_codeql_bundle_version="${prior_codeql_cli_version}-${prior_codeql_tag_name##*-}" -codeqlToolcachePath="$AGENT_TOOLSDIRECTORY/codeql/$codeql_bundle_version/x64" -mkdir -p $codeqlToolcachePath +# Download and name both CodeQL bundles. +codeql_bundle_versions=("${codeql_bundle_version}" "${prior_codeql_bundle_version}") +codeql_tag_names=("${codeql_tag_name}" "${prior_codeql_tag_name}") -echo "Unpacking the downloaded CodeQL bundle archive..." -tar -xzf $codeqlArchive -C $codeqlToolcachePath +for index in "${!codeql_bundle_versions[@]}"; do + bundle_version="${codeql_bundle_versions[$index]}" + bundle_tag_name="${codeql_tag_names[$index]}" + + echo "Downloading CodeQL bundle $bundle_version..." + download_with_retries "https://github.com/github/codeql-action/releases/download/$bundle_tag_name/codeql-bundle.tar.gz" "/tmp" "codeql-bundle.tar.gz" + codeql_archive="/tmp/codeql-bundle.tar.gz" -# Touch a special file that indicates to the CodeQL Action that this bundle was baked-in to the hosted runner images. -touch "$codeqlToolcachePath/pinned-version" + codeql_toolcache_path="$AGENT_TOOLSDIRECTORY/CodeQL/$bundle_version/x64" + mkdir -p "$codeql_toolcache_path" -# Touch a file to indicate to the toolcache that setting up CodeQL is complete. -touch "$codeqlToolcachePath.complete" + echo "Unpacking the downloaded CodeQL bundle archive..." + tar -xzf "$codeql_archive" -C "$codeql_toolcache_path" -invoke_tests "Common" "CodeQL" + # We only pin the latest version in the toolcache, to support overriding the CodeQL version specified in defaults.json on GitHub Enterprise. + if [[ "$bundle_version" == "$codeql_bundle_version" ]]; then + touch "$codeql_toolcache_path/pinned-version" + fi + + # Touch a file to indicate to the toolcache that setting up CodeQL is complete. + touch "$codeql_toolcache_path.complete" +done + +invoke_tests "Common" "CodeQLBundles" diff --git a/images/macos/software-report/SoftwareReport.Common.psm1 b/images/macos/software-report/SoftwareReport.Common.psm1 index 84035f13d..71568b5ee 100644 --- a/images/macos/software-report/SoftwareReport.Common.psm1 +++ b/images/macos/software-report/SoftwareReport.Common.psm1 @@ -632,12 +632,17 @@ function Build-GraalVMTable { } } -function Get-CodeQLBundleVersion { - $CodeQLVersionWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" - $CodeQLVersionPath = Get-ChildItem $CodeQLVersionWildcard | Select-Object -First 1 -Expand FullName - $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" - $CodeQLVersion = & $CodeQLPath version --quiet - return $CodeQLVersion +function Get-CodeQLBundleVersions { + $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" + $CodeQLVersionPaths = Get-ChildItem $CodeQLVersionsWildcard + $CodeQlVersions=@() + foreach ($CodeQLVersionPath in $CodeQLVersionPaths) { + $FullCodeQLVersionPath = $CodeQLVersionPath | Select-Object -Expand FullName + $CodeQLPath = Join-Path $FullCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" + $CodeQLVersion = & $CodeQLPath version --quiet + $CodeQLVersions += $CodeQLVersion + } + return $CodeQLVersions } function Get-ColimaVersion { diff --git a/images/macos/software-report/SoftwareReport.Generator.ps1 b/images/macos/software-report/SoftwareReport.Generator.ps1 index afc7b94b4..23e456a4a 100644 --- a/images/macos/software-report/SoftwareReport.Generator.ps1 +++ b/images/macos/software-report/SoftwareReport.Generator.ps1 @@ -137,7 +137,7 @@ $tools.AddToolVersion("Bicep CLI", $(Get-BicepVersion)) $tools.AddToolVersion("Cabal", $(Get-CabalVersion)) $tools.AddToolVersion("Cmake", $(Get-CmakeVersion)) if (-not $os.IsCatalina) { - $tools.AddToolVersion("CodeQL Action Bundle", $(Get-CodeQLBundleVersion)) + $tools.AddToolVersion("CodeQL Action Bundles", $(Get-CodeQLBundleVersions)) } if (-not $os.IsCatalina) { $tools.AddToolVersion("Colima", $(Get-ColimaVersion)) diff --git a/images/macos/tests/Common.Tests.ps1 b/images/macos/tests/Common.Tests.ps1 index ad1a19876..eb17db53a 100644 --- a/images/macos/tests/Common.Tests.ps1 +++ b/images/macos/tests/Common.Tests.ps1 @@ -139,15 +139,39 @@ Describe "VirtualBox" -Skip:($os.IsBigSur) { } } -Describe "CodeQL" -Skip:($os.IsCatalina) { - It "codeql" { - $CodeQLVersionWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" - $CodeQLVersionPath = Get-ChildItem $CodeQLVersionWildcard | Select-Object -First 1 -Expand FullName - $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" - "$CodeQLPath version --quiet" | Should -ReturnZeroExitCode +Describe "CodeQLBundles" -Skip:($os.IsCatalina) { + It "Latest CodeQL Bundle" { + $CodeQLVersionWildcards = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" + $LatestCodeQLVersionPath = Get-ChildItem $CodeQLVersionWildcards | Sort-Object -Property { [SemVer]$_.name } -Descending | Select-Object -First 1 -Expand FullName + $LatestCodeQLPath = Join-Path $LatestCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" + "$LatestCodeQLPath version --quiet" | Should -ReturnZeroExitCode - $CodeQLPacksPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "qlpacks" - $CodeQLPacksPath | Should -Exist + $LatestCodeQLPacksPath = Join-Path $LatestCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "qlpacks" + $LatestCodeQLPacksPath | Should -Exist + } + + It "Prior CodeQL Bundle" { + $CodeQLVersionWildcards = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" + $PriorCodeQLVersionPath = Get-ChildItem $CodeQLVersionWildcards | Sort-Object -Property { [SemVer]$_.name } -Descending | Select-Object -Last 1 -Expand FullName + $PriorCodeQLPath = Join-Path $PriorCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" + "$PriorCodeQLPath version --quiet" | Should -ReturnZeroExitCode + + $PriorCodeQLPacksPath = Join-Path $PriorCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "qlpacks" + $PriorCodeQLPacksPath | Should -Exist + } + + It "Latest and Prior CodeQL Bundles are unique" { + $CodeQLVersionWildcards = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" + + $LatestCodeQLVersionPath = Get-ChildItem $CodeQLVersionWildcards | Sort-Object -Property { [SemVer]$_.name } -Descending | Select-Object -First 1 -Expand FullName + $LatestCodeQLPath = Join-Path $LatestCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" + $LatestCodeQLVersion = & $LatestCodeQLPath version --quiet + + $PriorCodeQLVersionPath = Get-ChildItem $CodeQLVersionWildcards | Sort-Object -Property { [SemVer]$_.name } -Descending | Select-Object -Last 1 -Expand FullName + $PriorCodeQLPath = Join-Path $PriorCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" + $PriorCodeQLVersion = & $PriorCodeQLPath version --quiet + + $LatestCodeQLVersion | Should -Not -Match $PriorCodeQLVersion } } diff --git a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 index b53f7d015..6c0aeecf8 100644 --- a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 +++ b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 @@ -4,25 +4,48 @@ ################################################################################ # Retrieve the name of the CodeQL bundle preferred by the Action (in the format codeql-bundle-YYYYMMDD). -$CodeQLBundleName = (Invoke-RestMethod "https://raw.githubusercontent.com/github/codeql-action/v2/src/defaults.json").bundleVersion -# Convert the bundle name to a version number (0.0.0-YYYYMMDD). -$CodeQLBundleVersion = "0.0.0-" + $CodeQLBundleName.split("-")[-1] +$Defaults = (Invoke-RestMethod "https://raw.githubusercontent.com/github/codeql-action/v2/src/defaults.json") +$CodeQLTagName = $Defaults.bundleVersion +$CodeQLCliVersion = $Defaults.cliVersion +$PriorCodeQLTagName = $Defaults.priorBundleVersion +$PriorCodeQLCliVersion = $Defaults.priorCliVersion -$ExtractionDirectory = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath $CodeQLBundleVersion | Join-Path -ChildPath "x64" -New-Item -Path $ExtractionDirectory -ItemType Directory -Force | Out-Null +# Convert the tag names to bundles with a version number (x.y.z-YYYYMMDD). +$CodeQLBundleVersion = $CodeQLCliVersion + "-" + $CodeQLTagName.split("-")[-1] +$PriorCodeQLBundleVersion = $PriorCodeQLCliVersion + "-" + $PriorCodeQLTagName.split("-")[-1] -Write-Host "Downloading CodeQL bundle $CodeQLBundleVersion..." -$CodeQLBundlePath = Start-DownloadWithRetry -Url "https://github.com/github/codeql-action/releases/download/$CodeQLBundleName/codeql-bundle.tar.gz" -Name "codeql-bundle.tar.gz" -$DownloadDirectoryPath = (Get-Item $CodeQLBundlePath).Directory.FullName -Extract-7Zip -Path $CodeQLBundlePath -DestinationPath $DownloadDirectoryPath -$UnGzipedCodeQLBundlePath = Join-Path $DownloadDirectoryPath "codeql-bundle.tar" -Extract-7Zip -Path $UnGzipedCodeQLBundlePath -DestinationPath $ExtractionDirectory +$Bundles = @( + [PSCustomObject]@{ + TagName=$CodeQLTagName; + BundleVersion=$CodeQLBundleVersion; + }, + [PSCustomObject]@{ + TagName=$PriorCodeQLTagName; + BundleVersion=$PriorCodeQLBundleVersion; + } +) -# Touch a special file that indicates to the CodeQL Action that this bundle was baked-in to the hosted runner images. -New-Item -ItemType file (Join-Path $ExtractionDirectory -ChildPath "pinned-version") +foreach ($Bundle in $Bundles) { + Write-Host "Downloading CodeQL bundle $($Bundle.BundleVersion)..." + $CodeQLBundlePath = Start-DownloadWithRetry -Url "https://github.com/github/codeql-action/releases/download/$($Bundle.TagName)/codeql-bundle.tar.gz" -Name "codeql-bundle.tar.gz" + $DownloadDirectoryPath = (Get-Item $CodeQLBundlePath).Directory.FullName -# Touch a file to indicate to the toolcache that setting up CodeQL is complete. -New-Item -ItemType file "$ExtractionDirectory.complete" + $CodeQLToolcachePath = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath $Bundle.BundleVersion | Join-Path -ChildPath "x64" + New-Item -Path $CodeQLToolcachePath -ItemType Directory -Force | Out-Null -# Test that the tool has been extracted successfully. -Invoke-PesterTests -TestFile "Tools" -TestName "CodeQLBundle" + Write-Host "Unpacking the downloaded CodeQL bundle archive..." + Extract-7Zip -Path $CodeQLBundlePath -DestinationPath $DownloadDirectoryPath + $UnGzipedCodeQLBundlePath = Join-Path $DownloadDirectoryPath "codeql-bundle.tar" + Extract-7Zip -Path $UnGzipedCodeQLBundlePath -DestinationPath $CodeQLToolcachePath + + # We only pin the latest version in the toolcache, to support overriding the CodeQL version specified in defaults.json on GitHub Enterprise. + if ($Bundle.BundleVersion -eq $CodeQLBundleVersion) { + New-Item -ItemType file (Join-Path $CodeQLToolcachePath -ChildPath "pinned-version") + } + + # Touch a file to indicate to the toolcache that setting up CodeQL is complete. + New-Item -ItemType file "$CodeQLToolcachePath.complete" +} + +# Test that the tools have been extracted successfully. +Invoke-PesterTests -TestFile "Tools" -TestName "CodeQLBundles" diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 index a89db9cf3..a110f4361 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 @@ -69,7 +69,7 @@ $tools.AddToolVersion("Bazelisk", $(Get-BazeliskVersion)) $tools.AddToolVersion("Bicep", $(Get-BicepVersion)) $tools.AddToolVersion("Cabal", $(Get-CabalVersion)) $tools.AddToolVersion("CMake", $(Get-CMakeVersion)) -$tools.AddToolVersion("CodeQL Action Bundle", $(Get-CodeQLBundleVersion)) +$tools.AddToolVersion("CodeQL Action Bundle", $(Get-CodeQLBundleVersions)) $tools.AddToolVersion("Docker", $(Get-DockerVersion)) $tools.AddToolVersion("Docker Compose v1", $(Get-DockerComposeVersion)) $tools.AddToolVersion("Docker Compose v2", $(Get-DockerComposeVersionV2)) diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 b/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 index ef583cad9..b8837403e 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 @@ -42,12 +42,17 @@ function Get-CMakeVersion { return $cmakeVersion } -function Get-CodeQLBundleVersion { - $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "codeql" | Join-Path -ChildPath "*" - $CodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Select-Object -First 1 -Expand FullName - $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" - $CodeQLVersion = & $CodeQLPath version --quiet - return $CodeQLVersion +function Get-CodeQLBundleVersions { + $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" + $CodeQLVersionPaths = Get-ChildItem $CodeQLVersionsWildcard + $CodeQlVersions=@() + foreach ($CodeQLVersionPath in $CodeQLVersionPaths) { + $FullCodeQLVersionPath = $CodeQLVersionPath | Select-Object -Expand FullName + $CodeQLPath = Join-Path $FullCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" + $CodeQLVersion = & $CodeQLPath version --quiet + $CodeQLVersions += $CodeQLVersion + } + return $CodeQLVersions } function Get-DockerVersion { diff --git a/images/win/scripts/Tests/Tools.Tests.ps1 b/images/win/scripts/Tests/Tools.Tests.ps1 index 247e48bd8..873a41613 100644 --- a/images/win/scripts/Tests/Tools.Tests.ps1 +++ b/images/win/scripts/Tests/Tools.Tests.ps1 @@ -21,13 +21,41 @@ Describe "Bazel" { } } -Describe "CodeQLBundle" { - It "CodeQLBundle" { - $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "codeql" | Join-Path -ChildPath "*" - $CodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Select-Object -First 1 -Expand FullName - $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" - "$CodeQLPath version" | Should -ReturnZeroExitCode +Describe "CodeQLBundles" { + It "Latest CodeQL Bundle" { + $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" + $LatestCodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Sort-Object -Descending | Select-Object -First 1 -Expand FullName + $LatestCodeQLPath = Join-Path $LatestCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" + "$LatestCodeQLPath version --quiet" | Should -ReturnZeroExitCode + + $LatestCodeQLPacksPath = Join-Path $LatestCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "qlpacks" + $LatestCodeQLPacksPath | Should -Exist } + + It "Prior CodeQL Bundle" { + $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" + $PriorCodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Sort-Object -Descending | Select-Object -Last 1 -Expand FullName + $PriorCodeQLPath = Join-Path $PriorCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" + "$PriorCodeQLPath version --quiet" | Should -ReturnZeroExitCode + + $PriorCodeQLPacksPath = Join-Path $PriorCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "qlpacks" + $PriorCodeQLPacksPath | Should -Exist + } + + It "Latest and Prior CodeQL Bundles are unique" { + $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" + + $LatestCodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Sort-Object -Descending | Select-Object -First 1 -Expand FullName + $LatestCodeQLPath = Join-Path $LatestCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" + $LatestCodeQLVersion = & $LatestCodeQLPath version --quiet + + $PriorCodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Sort-Object -Descending | Select-Object -Last 1 -Expand FullName + $PriorCodeQLPath = Join-Path $PriorCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" + $PriorCodeQLVersion = & $PriorCodeQLPath version --quiet + + $LatestCodeQLVersion | Should -Not -Match $PriorCodeQLVersion + } + } Describe "R" {