From f2d892281b2389a22cbe1b509e6b0388f9993514 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Tue, 17 Oct 2023 21:18:48 +0100 Subject: [PATCH] macOS: Cache only the latest version of CodeQL (#8420) Previously, we cached two versions since we prioritized hitting the toolcache over landing new releases quicker. However after experimenting with this, we have decided to prioritize getting new releases into customers' hands more quickly. --- images/macos/provision/core/codeql-bundle.sh | 78 +++++-------------- .../SoftwareReport.Common.psm1 | 18 ++--- .../SoftwareReport.Generator.ps1 | 2 +- images/macos/tests/Common.Tests.ps1 | 40 ++-------- 4 files changed, 34 insertions(+), 104 deletions(-) diff --git a/images/macos/provision/core/codeql-bundle.sh b/images/macos/provision/core/codeql-bundle.sh index abce292e..032c18b2 100644 --- a/images/macos/provision/core/codeql-bundle.sh +++ b/images/macos/provision/core/codeql-bundle.sh @@ -1,70 +1,28 @@ #!/bin/bash -e -o pipefail source ~/utils/utils.sh -# Retrieve the CLI versions and bundle tags of the latest two CodeQL bundles. +# Retrieve the CLI version of the latest CodeQL bundle. base_url="$(curl -fsSL https://raw.githubusercontent.com/github/codeql-action/v2/src/defaults.json)" -codeql_tag_name="$(echo "$base_url" | jq -r '.bundleVersion')" -codeql_cli_version="$(echo "$base_url" | jq -r '.cliVersion')" -prior_codeql_tag_name="$(echo "$base_url" | jq -r '.priorBundleVersion')" -prior_codeql_cli_version="$(echo "$base_url" | jq -r '.priorCliVersion')" +bundle_version="$(echo "$base_url" | jq -r '.cliVersion')" +bundle_tag_name="codeql-bundle-v$bundle_version" -# Compute the toolcache version number for each bundle. This is either `x.y.z` or `x.y.z-YYYYMMDD`. -if [[ "${codeql_tag_name##*-}" == "v"* ]]; then - # Tag name of the format `codeql-bundle-vx.y.z`, where x.y.z is the CLI version. - # We don't need to include the tag name in the toolcache version number because it's derivable - # from the CLI version. - codeql_bundle_version="$codeql_cli_version" -elif [[ "${codeql_tag_name##*-}" =~ ^[0-9]+$ ]]; then - # Tag name of the format `codeql-bundle-YYYYMMDD`. - # We need to include the tag name in the toolcache version number because it can't be derived - # from the CLI version. - codeql_bundle_version="$codeql_cli_version-${codeql_tag_name##*-}" -else - echo "Unrecognised current CodeQL bundle tag name: $codeql_tag_name." \ - "Could not compute toolcache version number." - exit 1 -fi -if [[ "${prior_codeql_tag_name##*-}" == "v"* ]]; then - # Tag name of the format `codeql-bundle-vx.y.z`, where x.y.z is the CLI version. - # We don't need to include the tag name in the toolcache version number because it's derivable - # from the CLI version. - prior_codeql_bundle_version="$prior_codeql_cli_version" -elif [[ "${prior_codeql_tag_name##*-}" =~ ^[0-9]+$ ]]; then - # Tag name of the format `codeql-bundle-YYYYMMDD`. - # We need to include the tag name in the toolcache version number because it can't be derived - # from the CLI version. - prior_codeql_bundle_version="$prior_codeql_cli_version-${prior_codeql_tag_name##*-}" -else - echo "Unrecognised prior CodeQL bundle tag name: $prior_codeql_tag_name." \ - "Could not compute toolcache version number." - exit 1 -fi +echo "Downloading CodeQL bundle $bundle_version..." +# Note that this is the all-platforms CodeQL bundle, to support scenarios where customers run +# different operating systems within containers. +download_with_retries "https://github.com/github/codeql-action/releases/download/$bundle_tag_name/codeql-bundle.tar.gz" "/tmp" "codeql-bundle.tar.gz" +codeql_archive="/tmp/codeql-bundle.tar.gz" -# Download and name both CodeQL bundles. -codeql_bundle_versions=("${codeql_bundle_version}" "${prior_codeql_bundle_version}") -codeql_tag_names=("${codeql_tag_name}" "${prior_codeql_tag_name}") +codeql_toolcache_path="$AGENT_TOOLSDIRECTORY/CodeQL/$bundle_version/x64" +mkdir -p "$codeql_toolcache_path" -for index in "${!codeql_bundle_versions[@]}"; do - bundle_version="${codeql_bundle_versions[$index]}" - bundle_tag_name="${codeql_tag_names[$index]}" - - echo "Downloading CodeQL bundle $bundle_version..." - download_with_retries "https://github.com/github/codeql-action/releases/download/$bundle_tag_name/codeql-bundle.tar.gz" "/tmp" "codeql-bundle.tar.gz" - codeql_archive="/tmp/codeql-bundle.tar.gz" +echo "Unpacking the downloaded CodeQL bundle archive..." +tar -xzf "$codeql_archive" -C "$codeql_toolcache_path" - codeql_toolcache_path="$AGENT_TOOLSDIRECTORY/CodeQL/$bundle_version/x64" - mkdir -p "$codeql_toolcache_path" +# Touch a file to indicate to the CodeQL Action that this bundle shipped with the toolcache. This is +# to support overriding the CodeQL version specified in defaults.json on GitHub Enterprise. +touch "$codeql_toolcache_path/pinned-version" - echo "Unpacking the downloaded CodeQL bundle archive..." - tar -xzf "$codeql_archive" -C "$codeql_toolcache_path" +# Touch a file to indicate to the toolcache that setting up CodeQL is complete. +touch "$codeql_toolcache_path.complete" - # We only pin the latest version in the toolcache, to support overriding the CodeQL version specified in defaults.json on GitHub Enterprise. - if [[ "$bundle_version" == "$codeql_bundle_version" ]]; then - touch "$codeql_toolcache_path/pinned-version" - fi - - # Touch a file to indicate to the toolcache that setting up CodeQL is complete. - touch "$codeql_toolcache_path.complete" -done - -invoke_tests "Common" "CodeQLBundles" +invoke_tests "Common" "CodeQL Bundle" diff --git a/images/macos/software-report/SoftwareReport.Common.psm1 b/images/macos/software-report/SoftwareReport.Common.psm1 index beaaa30a..29d2e9db 100644 --- a/images/macos/software-report/SoftwareReport.Common.psm1 +++ b/images/macos/software-report/SoftwareReport.Common.psm1 @@ -608,17 +608,13 @@ function Build-MiscellaneousEnvironmentTable { } } -function Get-CodeQLBundleVersions { - $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" - $CodeQLVersionPaths = Get-ChildItem $CodeQLVersionsWildcard - $CodeQlVersions=@() - foreach ($CodeQLVersionPath in $CodeQLVersionPaths) { - $FullCodeQLVersionPath = $CodeQLVersionPath | Select-Object -Expand FullName - $CodeQLPath = Join-Path $FullCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" - $CodeQLVersion = & $CodeQLPath version --quiet - $CodeQLVersions += $CodeQLVersion - } - return $CodeQLVersions + +function Get-CodeQLBundleVersion { + $CodeQLVersionWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" + $CodeQLVersionPath = Get-ChildItem $CodeQLVersionWildcard | Select-Object -First 1 -Expand FullName + $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" + $CodeQLVersion = & $CodeQLPath version --quiet + return $CodeQLVersion } function Get-ColimaVersion { diff --git a/images/macos/software-report/SoftwareReport.Generator.ps1 b/images/macos/software-report/SoftwareReport.Generator.ps1 index c94590cd..16056558 100644 --- a/images/macos/software-report/SoftwareReport.Generator.ps1 +++ b/images/macos/software-report/SoftwareReport.Generator.ps1 @@ -172,7 +172,7 @@ if (-not $os.IsVenturaArm64) { $tools.AddToolVersion("Cabal", $(Get-CabalVersion)) } $tools.AddToolVersion("Cmake", $(Get-CmakeVersion)) -$tools.AddToolVersion("CodeQL Action Bundles", $(Get-CodeQLBundleVersions)) +$tools.AddToolVersion("CodeQL Action Bundle", $(Get-CodeQLBundleVersion)) if ($os.IsMonterey) { $tools.AddToolVersion("Colima", $(Get-ColimaVersion)) } diff --git a/images/macos/tests/Common.Tests.ps1 b/images/macos/tests/Common.Tests.ps1 index 1ac30ee4..e1e645e7 100644 --- a/images/macos/tests/Common.Tests.ps1 +++ b/images/macos/tests/Common.Tests.ps1 @@ -129,39 +129,15 @@ Describe "VirtualBox" -Skip:($os.IsBigSur -or $os.IsVentura -or $os.IsVenturaArm } } -Describe "CodeQLBundles" { - It "Latest CodeQL Bundle" { - $CodeQLVersionWildcards = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" - $LatestCodeQLVersionPath = Get-ChildItem $CodeQLVersionWildcards | Sort-Object -Property { [SemVer]$_.name } -Descending | Select-Object -First 1 -Expand FullName - $LatestCodeQLPath = Join-Path $LatestCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" - "$LatestCodeQLPath version --quiet" | Should -ReturnZeroExitCode +Describe "CodeQL Bundle" { + It "Is installed" { + $CodeQLVersionWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" + $CodeQLVersionPath = Get-ChildItem $CodeQLVersionWildcard | Select-Object -First 1 -Expand FullName + $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" + "$CodeQLPath version --quiet" | Should -ReturnZeroExitCode - $LatestCodeQLPacksPath = Join-Path $LatestCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "qlpacks" - $LatestCodeQLPacksPath | Should -Exist - } - - It "Prior CodeQL Bundle" { - $CodeQLVersionWildcards = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" - $PriorCodeQLVersionPath = Get-ChildItem $CodeQLVersionWildcards | Sort-Object -Property { [SemVer]$_.name } -Descending | Select-Object -Last 1 -Expand FullName - $PriorCodeQLPath = Join-Path $PriorCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" - "$PriorCodeQLPath version --quiet" | Should -ReturnZeroExitCode - - $PriorCodeQLPacksPath = Join-Path $PriorCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "qlpacks" - $PriorCodeQLPacksPath | Should -Exist - } - - It "Latest and Prior CodeQL Bundles are unique" { - $CodeQLVersionWildcards = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath "*" - - $LatestCodeQLVersionPath = Get-ChildItem $CodeQLVersionWildcards | Sort-Object -Property { [SemVer]$_.name } -Descending | Select-Object -First 1 -Expand FullName - $LatestCodeQLPath = Join-Path $LatestCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" - $LatestCodeQLVersion = & $LatestCodeQLPath version --quiet - - $PriorCodeQLVersionPath = Get-ChildItem $CodeQLVersionWildcards | Sort-Object -Property { [SemVer]$_.name } -Descending | Select-Object -Last 1 -Expand FullName - $PriorCodeQLPath = Join-Path $PriorCodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" - $PriorCodeQLVersion = & $PriorCodeQLPath version --quiet - - $LatestCodeQLVersion | Should -Not -Match $PriorCodeQLVersion + $CodeQLPacksPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "qlpacks" + $CodeQLPacksPath | Should -Exist } }