From 5669edde7597b63910572942fbc688fa3fa825fe Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Wed, 4 Oct 2023 15:31:00 +0100 Subject: [PATCH] [Windows] Cache only the latest version of CodeQL (#8421) * Windows: Cache only the latest version of CodeQL Previously, we cached two versions since we prioritized hitting the toolcache over landing new releases quicker. However after experimenting with this, we have decided to prioritize getting new releases into customers' hands more quickly. * Break Windows tests down into separate assertions * List contents of bundle after extracting --- .../Installers/Install-CodeQLBundle.ps1 | 85 +++++-------------- .../SoftwareReport.Generator.ps1 | 2 +- .../SoftwareReport/SoftwareReport.Tools.psm1 | 15 ++-- images/win/scripts/Tests/Tools.Tests.ps1 | 38 +++------ 4 files changed, 39 insertions(+), 101 deletions(-) diff --git a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 index 7a063cc2e..b21977a20 100644 --- a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 +++ b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 @@ -3,75 +3,34 @@ ## Desc: Install the CodeQL CLI Bundle to the toolcache. ################################################################################ -# Retrieve the CLI versions and bundle tags of the latest two CodeQL bundles. +# Retrieve the CLI version of the latest CodeQL bundle. $Defaults = (Invoke-RestMethod "https://raw.githubusercontent.com/github/codeql-action/v2/src/defaults.json") -$CodeQLTagName = $Defaults.bundleVersion -$CodeQLCliVersion = $Defaults.cliVersion -$PriorCodeQLTagName = $Defaults.priorBundleVersion -$PriorCodeQLCliVersion = $Defaults.priorCliVersion +$CliVersion = $Defaults.cliVersion +$TagName = "codeql-bundle-v" + $CliVersion -# Compute the toolcache version number for each bundle. This is either `x.y.z` or `x.y.z-YYYYMMDD`. -if ($CodeQLTagName.split("-")[-1].StartsWith("v")) { - # Tag name of the format `codeql-bundle-vx.y.z`, where x.y.z is the CLI version. - # We don't need to include the tag name in the toolcache version number because it's derivable - # from the CLI version. - $CodeQLBundleVersion = $CodeQLCliVersion -} elseif ($CodeQLTagName.split("-")[-1] -match "^\d+$") { - # Tag name of the format `codeql-bundle-YYYYMMDD`. - # We need to include the tag name in the toolcache version number because it can't be derived - # from the CLI version. - $CodeQLBundleVersion = $CodeQLCliVersion + "-" + $CodeQLTagName.split("-")[-1] -} else { - Write-Error "Unrecognised current CodeQL bundle tag name: $CodeQLTagName. Could not compute toolcache version number." - exit 1 -} -if ($PriorCodeQLTagName.split("-")[-1].StartsWith("v")) { - # Tag name of the format `codeql-bundle-vx.y.z`, where x.y.z is the CLI version. - # We don't need to include the tag name in the toolcache version number because it's derivable - # from the CLI version. - $PriorCodeQLBundleVersion = $PriorCodeQLCliVersion -} elseif ($PriorCodeQLTagName.split("-")[-1] -match "^\d+$") { - # Tag name of the format `codeql-bundle-YYYYMMDD`. - # We need to include the tag name in the toolcache version number because it can't be derived - # from the CLI version. - $PriorCodeQLBundleVersion = $PriorCodeQLCliVersion + "-" + $PriorCodeQLTagName.split("-")[-1] -} else { - Write-Error "Unrecognised prior CodeQL bundle tag name: $PriorCodeQLTagName. Could not compute toolcache version number." - exit 1 -} +Write-Host "Downloading CodeQL bundle $($CliVersion)..." +# Note that this is the all-platforms CodeQL bundle, to support scenarios where customers run +# different operating systems within containers. +$CodeQLBundlePath = Start-DownloadWithRetry -Url "https://github.com/github/codeql-action/releases/download/$($TagName)/codeql-bundle.tar.gz" -Name "codeql-bundle.tar.gz" +$DownloadDirectoryPath = (Get-Item $CodeQLBundlePath).Directory.FullName -$Bundles = @( - [PSCustomObject]@{ - TagName=$CodeQLTagName; - BundleVersion=$CodeQLBundleVersion; - }, - [PSCustomObject]@{ - TagName=$PriorCodeQLTagName; - BundleVersion=$PriorCodeQLBundleVersion; - } -) +$CodeQLToolcachePath = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath $CliVersion | Join-Path -ChildPath "x64" +New-Item -Path $CodeQLToolcachePath -ItemType Directory -Force | Out-Null -foreach ($Bundle in $Bundles) { - Write-Host "Downloading CodeQL bundle $($Bundle.BundleVersion)..." - $CodeQLBundlePath = Start-DownloadWithRetry -Url "https://github.com/github/codeql-action/releases/download/$($Bundle.TagName)/codeql-bundle.tar.gz" -Name "codeql-bundle.tar.gz" - $DownloadDirectoryPath = (Get-Item $CodeQLBundlePath).Directory.FullName +Write-Host "Unpacking the downloaded CodeQL bundle archive..." +Extract-7Zip -Path $CodeQLBundlePath -DestinationPath $DownloadDirectoryPath +$UnGzipedCodeQLBundlePath = Join-Path $DownloadDirectoryPath "codeql-bundle.tar" +Extract-7Zip -Path $UnGzipedCodeQLBundlePath -DestinationPath $CodeQLToolcachePath - $CodeQLToolcachePath = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath $Bundle.BundleVersion | Join-Path -ChildPath "x64" - New-Item -Path $CodeQLToolcachePath -ItemType Directory -Force | Out-Null +Write-Host "CodeQL bundle at $($CodeQLToolcachePath) contains the following directories:" +Get-ChildItem -Path $CodeQLToolcachePath -Depth 2 - Write-Host "Unpacking the downloaded CodeQL bundle archive..." - Extract-7Zip -Path $CodeQLBundlePath -DestinationPath $DownloadDirectoryPath - $UnGzipedCodeQLBundlePath = Join-Path $DownloadDirectoryPath "codeql-bundle.tar" - Extract-7Zip -Path $UnGzipedCodeQLBundlePath -DestinationPath $CodeQLToolcachePath +# Touch a file to indicate to the CodeQL Action that this bundle shipped with the toolcache. This is +# to support overriding the CodeQL version specified in defaults.json on GitHub Enterprise. +New-Item -ItemType file (Join-Path $CodeQLToolcachePath -ChildPath "pinned-version") - # We only pin the latest version in the toolcache, to support overriding the CodeQL version specified in defaults.json on GitHub Enterprise. - if ($Bundle.BundleVersion -eq $CodeQLBundleVersion) { - New-Item -ItemType file (Join-Path $CodeQLToolcachePath -ChildPath "pinned-version") - } - - # Touch a file to indicate to the toolcache that setting up CodeQL is complete. - New-Item -ItemType file "$CodeQLToolcachePath.complete" -} +# Touch a file to indicate to the toolcache that setting up CodeQL is complete. +New-Item -ItemType file "$CodeQLToolcachePath.complete" # Test that the tools have been extracted successfully. -Invoke-PesterTests -TestFile "Tools" -TestName "CodeQLBundles" +Invoke-PesterTests -TestFile "Tools" -TestName "CodeQL Bundle" diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 index dcd76b889..70c0021eb 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 @@ -69,7 +69,7 @@ $tools.AddToolVersion("Bazelisk", $(Get-BazeliskVersion)) $tools.AddToolVersion("Bicep", $(Get-BicepVersion)) $tools.AddToolVersion("Cabal", $(Get-CabalVersion)) $tools.AddToolVersion("CMake", $(Get-CMakeVersion)) -$tools.AddToolVersion("CodeQL Action Bundles", $(Get-CodeQLBundleVersions)) +$tools.AddToolVersion("CodeQL Action Bundle", $(Get-CodeQLBundleVersion)) $tools.AddToolVersion("Docker", $(Get-DockerVersion)) $tools.AddToolVersion("Docker Compose v1", $(Get-DockerComposeVersion)) $tools.AddToolVersion("Docker Compose v2", $(Get-DockerComposeVersionV2)) diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 b/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 index b1038167e..ce997c84f 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 @@ -42,17 +42,12 @@ function Get-CMakeVersion { return $cmakeVersion } -function Get-CodeQLBundleVersions { +function Get-CodeQLBundleVersion { $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" - $CodeQLVersionPaths = Get-ChildItem $CodeQLVersionsWildcard - $CodeQlVersions=@() - foreach ($CodeQLVersionPath in $CodeQLVersionPaths) { - $FullCodeQLVersionPath = $CodeQLVersionPath | Select-Object -Expand FullName - $CodeQLPath = Join-Path $FullCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" - $CodeQLVersion = & $CodeQLPath version --quiet - $CodeQLVersions += $CodeQLVersion - } - return $CodeQLVersions + $CodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Select-Object -First 1 -Expand FullName + $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" + $CodeQLVersion = & $CodeQLPath version --quiet + return $CodeQLVersion } function Get-DockerVersion { diff --git a/images/win/scripts/Tests/Tools.Tests.ps1 b/images/win/scripts/Tests/Tools.Tests.ps1 index 9c91ac55b..be8864d8b 100644 --- a/images/win/scripts/Tests/Tools.Tests.ps1 +++ b/images/win/scripts/Tests/Tools.Tests.ps1 @@ -21,41 +21,25 @@ Describe "Bazel" { } } -Describe "CodeQLBundles" { - It "Latest CodeQL Bundle" { +Describe "CodeQL Bundle" { + It "Single distribution installed" { $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" - $LatestCodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Sort-Object -Descending | Select-Object -First 1 -Expand FullName - $LatestCodeQLPath = Join-Path $LatestCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" - "$LatestCodeQLPath version --quiet" | Should -ReturnZeroExitCode - - $LatestCodeQLPacksPath = Join-Path $LatestCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "qlpacks" - $LatestCodeQLPacksPath | Should -Exist + $CodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Should -HaveCount 1 } - It "Prior CodeQL Bundle" { + It "Contains CodeQL executable" { $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" - $PriorCodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Sort-Object -Descending | Select-Object -Last 1 -Expand FullName - $PriorCodeQLPath = Join-Path $PriorCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" - "$PriorCodeQLPath version --quiet" | Should -ReturnZeroExitCode - - $PriorCodeQLPacksPath = Join-Path $PriorCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "qlpacks" - $PriorCodeQLPacksPath | Should -Exist + $CodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Sort-Object -Descending | Select-Object -First 1 -Expand FullName + $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" + "$CodeQLPath version --quiet" | Should -ReturnZeroExitCode } - It "Latest and Prior CodeQL Bundles are unique" { + It "Contains CodeQL packs" { $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" - - $LatestCodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Sort-Object -Descending | Select-Object -First 1 -Expand FullName - $LatestCodeQLPath = Join-Path $LatestCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" - $LatestCodeQLVersion = & $LatestCodeQLPath version --quiet - - $PriorCodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Sort-Object -Descending | Select-Object -Last 1 -Expand FullName - $PriorCodeQLPath = Join-Path $PriorCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" - $PriorCodeQLVersion = & $PriorCodeQLPath version --quiet - - $LatestCodeQLVersion | Should -Not -Match $PriorCodeQLVersion + $CodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Sort-Object -Descending | Select-Object -First 1 -Expand FullName + $CodeQLPacksPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "qlpacks" + $CodeQLPacksPath | Should -Exist } - } Describe "R" {