From 09dcae0f24d4b0ea85b4c9ae637d0f4aa065215f Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Tue, 8 Sep 2020 10:59:26 +0100 Subject: [PATCH 01/20] Install the CodeQL bundle in the toolcache. --- .../linux/scripts/installers/codeql-bundle.sh | 19 ++++++++++++++ .../Installers/Install-CodeQLBundle.ps1 | 26 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 images/linux/scripts/installers/codeql-bundle.sh create mode 100644 images/win/scripts/Installers/Install-CodeQLBundle.ps1 diff --git a/images/linux/scripts/installers/codeql-bundle.sh b/images/linux/scripts/installers/codeql-bundle.sh new file mode 100644 index 00000000..37e89abd --- /dev/null +++ b/images/linux/scripts/installers/codeql-bundle.sh @@ -0,0 +1,19 @@ +#!/bin/bash +################################################################################ +## File: codeql-bundle.sh +## Desc: Install the CodeQL CLI Bundle to the toolcache. +################################################################################ + +# Retrieve the name of the CodeQL bundle preferred by the Action (in the format codeql-bundle-YYYYMMDD). +codeql_bundle_name="$(curl -sSL https://raw.githubusercontent.com/github/codeql-action/main/src/defaults.json | jq -r .bundleVersion)" +# Convert the bundle name to a version number (0.0.0-YYYYMMDD). +codeql_bundle_version="0.0.0-${codeql_bundle_name##*-}" + +extraction_directory="$AGENT_TOOLSDIRECTORY/CodeQL/$codeql_bundle_version/x64" +mkdir -p "$extraction_directory" + +>&2 echo "Downloading CodeQL bundle $codeql_bundle_version..." +curl -sSL "https://github.com/github/codeql-action/releases/download/$codeql_bundle_name/codeql-bundle.tar.gz" | tar -xzC "$extraction_directory" + +# Test that the tool has been extracted successfully. +"$AGENT_TOOLSDIRECTORY/CodeQL/$codeql_bundle_version/x64/codeql/codeql" version diff --git a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 new file mode 100644 index 00000000..6f25fa46 --- /dev/null +++ b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 @@ -0,0 +1,26 @@ +################################################################################ +## File: Install-CodeQLBundle.ps1 +## Desc: Install the CodeQL CLI Bundle to the toolcache. +################################################################################ + +Import-Module -Name ImageHelpers + +# Retrieve the name of the CodeQL bundle preferred by the Action (in the format codeql-bundle-YYYYMMDD). +$CodeQLBundleName = (Invoke-WebRequest "https://raw.githubusercontent.com/github/codeql-action/main/src/defaults.json" | ConvertFrom-Json).bundleVersion +# Convert the bundle name to a version number (0.0.0-YYYYMMDD). +$CodeQLBundleVersion = "0.0.0-" + $CodeQLBundleName.split("-")[-1] + +$ExtractionDirectory = "$Env:AGENT_TOOLSDIRECTORY/CodeQL/$CodeQLBundleVersion/x64" +New-Item -Path $ExtractionDirectory -ItemType Directory -Force | Out-Null + +Write-Host "Downloading CodeQL bundle $CodeQLBundleVersion..." +$CodeQLBundlePath = Start-DownloadWithRetry -Url "https://github.com/github/codeql-action/releases/download/$CodeQLBundleName/codeql-bundle.tar.gz" -Name "codeql-bundle.tar.gz" +$DownloadDirectoryPath = (Get-Item $CodeQLBundlePath).Directory.FullName +Extract-7Zip -Path $CodeQLBundlePath -DestinationPath $DownloadDirectoryPath +Remove-Item -Path $CodeQLBundlePath +$UnGzipedCodeQLBundlePath = (Join-Path $DownloadDirectoryPath "codeql-bundle.tar") +Extract-7Zip -Path $UnGzipedCodeQLBundlePath -DestinationPath $ExtractionDirectory +Remove-Item -Path $UnGzipedCodeQLBundlePath + +# Test that the tool has been extracted successfully. +& (Join-Path $ExtractionDirectory "codeql" "codeql.exe") version From 79c4eb4910982ee20c0ce46a4172cd36db91a406 Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Tue, 8 Sep 2020 15:16:23 +0100 Subject: [PATCH 02/20] Tidy up CodeQL bundle Windows installer script in response to review comments. --- images/win/scripts/Installers/Install-CodeQLBundle.ps1 | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 index 6f25fa46..7878d6c6 100644 --- a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 +++ b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 @@ -3,24 +3,20 @@ ## Desc: Install the CodeQL CLI Bundle to the toolcache. ################################################################################ -Import-Module -Name ImageHelpers - # Retrieve the name of the CodeQL bundle preferred by the Action (in the format codeql-bundle-YYYYMMDD). -$CodeQLBundleName = (Invoke-WebRequest "https://raw.githubusercontent.com/github/codeql-action/main/src/defaults.json" | ConvertFrom-Json).bundleVersion +$CodeQLBundleName = (Invoke-RestMethod "https://raw.githubusercontent.com/github/codeql-action/main/src/defaults.json").bundleVersion # Convert the bundle name to a version number (0.0.0-YYYYMMDD). $CodeQLBundleVersion = "0.0.0-" + $CodeQLBundleName.split("-")[-1] -$ExtractionDirectory = "$Env:AGENT_TOOLSDIRECTORY/CodeQL/$CodeQLBundleVersion/x64" +$ExtractionDirectory = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "CodeQL" | Join-Path -ChildPath $CodeQLBundleVersion | Join-Path -ChildPath "x64" New-Item -Path $ExtractionDirectory -ItemType Directory -Force | Out-Null Write-Host "Downloading CodeQL bundle $CodeQLBundleVersion..." $CodeQLBundlePath = Start-DownloadWithRetry -Url "https://github.com/github/codeql-action/releases/download/$CodeQLBundleName/codeql-bundle.tar.gz" -Name "codeql-bundle.tar.gz" $DownloadDirectoryPath = (Get-Item $CodeQLBundlePath).Directory.FullName Extract-7Zip -Path $CodeQLBundlePath -DestinationPath $DownloadDirectoryPath -Remove-Item -Path $CodeQLBundlePath -$UnGzipedCodeQLBundlePath = (Join-Path $DownloadDirectoryPath "codeql-bundle.tar") +$UnGzipedCodeQLBundlePath = Join-Path $DownloadDirectoryPath "codeql-bundle.tar" Extract-7Zip -Path $UnGzipedCodeQLBundlePath -DestinationPath $ExtractionDirectory -Remove-Item -Path $UnGzipedCodeQLBundlePath # Test that the tool has been extracted successfully. & (Join-Path $ExtractionDirectory "codeql" "codeql.exe") version From 096464f38eb8597540566cead683c8a72b65a8f6 Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Tue, 8 Sep 2020 15:23:49 +0100 Subject: [PATCH 03/20] Tidy up CodeQL bundle Linux installer script in response to review comments. --- images/linux/scripts/installers/codeql-bundle.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/images/linux/scripts/installers/codeql-bundle.sh b/images/linux/scripts/installers/codeql-bundle.sh index 37e89abd..60008963 100644 --- a/images/linux/scripts/installers/codeql-bundle.sh +++ b/images/linux/scripts/installers/codeql-bundle.sh @@ -4,6 +4,8 @@ ## Desc: Install the CodeQL CLI Bundle to the toolcache. ################################################################################ +source $HELPER_SCRIPTS/install.sh + # Retrieve the name of the CodeQL bundle preferred by the Action (in the format codeql-bundle-YYYYMMDD). codeql_bundle_name="$(curl -sSL https://raw.githubusercontent.com/github/codeql-action/main/src/defaults.json | jq -r .bundleVersion)" # Convert the bundle name to a version number (0.0.0-YYYYMMDD). @@ -12,8 +14,9 @@ codeql_bundle_version="0.0.0-${codeql_bundle_name##*-}" extraction_directory="$AGENT_TOOLSDIRECTORY/CodeQL/$codeql_bundle_version/x64" mkdir -p "$extraction_directory" ->&2 echo "Downloading CodeQL bundle $codeql_bundle_version..." -curl -sSL "https://github.com/github/codeql-action/releases/download/$codeql_bundle_name/codeql-bundle.tar.gz" | tar -xzC "$extraction_directory" +echo "Downloading CodeQL bundle $codeql_bundle_version..." +download_with_retries "https://github.com/github/codeql-action/releases/download/$codeql_bundle_name/codeql-bundle.tar.gz" "/tmp" "codeql-bundle.tar.gz" +tar -xzf "/tmp/codeql-bundle.tar.gz" -C "$extraction_directory" # Test that the tool has been extracted successfully. "$AGENT_TOOLSDIRECTORY/CodeQL/$codeql_bundle_version/x64/codeql/codeql" version From ff346a80bd28bc6eb71dcbc27c8fb97e05bbd523 Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Tue, 8 Sep 2020 15:27:27 +0100 Subject: [PATCH 04/20] Actually call CodeQL bundle install scripts in Packer. --- images/linux/ubuntu1604.json | 1 + images/linux/ubuntu1804.json | 1 + images/linux/ubuntu2004.json | 1 + images/win/Windows2016-Azure.json | 3 ++- images/win/Windows2019-Azure.json | 3 ++- 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/images/linux/ubuntu1604.json b/images/linux/ubuntu1604.json index cf7aef1a..93161553 100644 --- a/images/linux/ubuntu1604.json +++ b/images/linux/ubuntu1604.json @@ -153,6 +153,7 @@ "{{template_dir}}/scripts/installers/clang.sh", "{{template_dir}}/scripts/installers/swift.sh", "{{template_dir}}/scripts/installers/cmake.sh", + "{{template_dir}}/scripts/installers/codeql-bundle.sh", "{{template_dir}}/scripts/installers/docker-compose.sh", "{{template_dir}}/scripts/installers/docker-moby.sh", "{{template_dir}}/scripts/installers/dotnetcore-sdk.sh", diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index 956e6773..c8e3da10 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -155,6 +155,7 @@ "{{template_dir}}/scripts/installers/clang.sh", "{{template_dir}}/scripts/installers/swift.sh", "{{template_dir}}/scripts/installers/cmake.sh", + "{{template_dir}}/scripts/installers/codeql-bundle.sh", "{{template_dir}}/scripts/installers/containers.sh", "{{template_dir}}/scripts/installers/docker-compose.sh", "{{template_dir}}/scripts/installers/docker-moby.sh", diff --git a/images/linux/ubuntu2004.json b/images/linux/ubuntu2004.json index 8a1d2d61..6f472c2d 100644 --- a/images/linux/ubuntu2004.json +++ b/images/linux/ubuntu2004.json @@ -157,6 +157,7 @@ "{{template_dir}}/scripts/installers/clang.sh", "{{template_dir}}/scripts/installers/swift.sh", "{{template_dir}}/scripts/installers/cmake.sh", + "{{template_dir}}/scripts/installers/codeql-bundle.sh", "{{template_dir}}/scripts/installers/containers.sh", "{{template_dir}}/scripts/installers/docker-compose.sh", "{{template_dir}}/scripts/installers/docker-moby.sh", diff --git a/images/win/Windows2016-Azure.json b/images/win/Windows2016-Azure.json index af87f5e8..39dcb254 100644 --- a/images/win/Windows2016-Azure.json +++ b/images/win/Windows2016-Azure.json @@ -320,7 +320,8 @@ "{{ template_dir }}/scripts/Installers/Install-KubernetesCli.ps1", "{{ template_dir }}/scripts/Installers/Install-Kind.ps1", "{{ template_dir }}/scripts/Installers/Install-MongoDB.ps1", - "{{ template_dir }}/scripts/Installers/Install-GoogleCloudSDK.ps1" + "{{ template_dir }}/scripts/Installers/Install-GoogleCloudSDK.ps1", + "{{ template_dir }}/scripts/Installers/Install-CodeQLBundle.ps1" ] }, { diff --git a/images/win/Windows2019-Azure.json b/images/win/Windows2019-Azure.json index 85c72206..a261ab17 100644 --- a/images/win/Windows2019-Azure.json +++ b/images/win/Windows2019-Azure.json @@ -319,7 +319,8 @@ "{{ template_dir }}/scripts/Installers/Install-AliyunCli.ps1", "{{ template_dir }}/scripts/Installers/Install-RootCA.ps1", "{{ template_dir }}/scripts/Installers/Install-MongoDB.ps1", - "{{ template_dir }}/scripts/Installers/Install-GoogleCloudSDK.ps1" + "{{ template_dir }}/scripts/Installers/Install-GoogleCloudSDK.ps1", + "{{ template_dir }}/scripts/Installers/Install-CodeQLBundle.ps1" ] }, { From 81942c8299ade2a7c017362711dbca501d217db1 Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Wed, 9 Sep 2020 15:50:49 +0100 Subject: [PATCH 05/20] Convert CodeQL Windows test to a Pester test. --- images/win/scripts/Installers/Install-CodeQLBundle.ps1 | 3 ++- images/win/scripts/Tests/Tools.Tests.ps1 | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 index 7878d6c6..d5ecf1b5 100644 --- a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 +++ b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 @@ -19,4 +19,5 @@ $UnGzipedCodeQLBundlePath = Join-Path $DownloadDirectoryPath "codeql-bundle.tar" Extract-7Zip -Path $UnGzipedCodeQLBundlePath -DestinationPath $ExtractionDirectory # Test that the tool has been extracted successfully. -& (Join-Path $ExtractionDirectory "codeql" "codeql.exe") version +$Env:CODEQL_EXTRACTION_DIRECTORY = $ExtractionDirectory +Invoke-PesterTests -TestFile "Tools" -TestName "CodeQLBundle" diff --git a/images/win/scripts/Tests/Tools.Tests.ps1 b/images/win/scripts/Tests/Tools.Tests.ps1 index 88f2ca88..c2b52baf 100644 --- a/images/win/scripts/Tests/Tools.Tests.ps1 +++ b/images/win/scripts/Tests/Tools.Tests.ps1 @@ -41,6 +41,13 @@ Describe "CMake" { } } +Describe "CodeQLBundle" { + It "CodeQLBundle" { + $CodeQLPath = Join-Path $Env:CODEQL_EXTRACTION_DIRECTORY -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" + "$CodeQLPath version" | Should -ReturnZeroExitCode + } +} + Describe "R" { It "Rscript" { "Rscript --version" | Should -ReturnZeroExitCode From 9337c6edb44c72f0fc1874ea834025b0c6f5be09 Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Tue, 15 Sep 2020 13:54:28 +0100 Subject: [PATCH 06/20] Fix not being able to find CodeQL. --- images/win/scripts/Installers/Install-CodeQLBundle.ps1 | 1 - images/win/scripts/Tests/Tools.Tests.ps1 | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 index d5ecf1b5..46a59759 100644 --- a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 +++ b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 @@ -19,5 +19,4 @@ $UnGzipedCodeQLBundlePath = Join-Path $DownloadDirectoryPath "codeql-bundle.tar" Extract-7Zip -Path $UnGzipedCodeQLBundlePath -DestinationPath $ExtractionDirectory # Test that the tool has been extracted successfully. -$Env:CODEQL_EXTRACTION_DIRECTORY = $ExtractionDirectory Invoke-PesterTests -TestFile "Tools" -TestName "CodeQLBundle" diff --git a/images/win/scripts/Tests/Tools.Tests.ps1 b/images/win/scripts/Tests/Tools.Tests.ps1 index c2b52baf..4470dda0 100644 --- a/images/win/scripts/Tests/Tools.Tests.ps1 +++ b/images/win/scripts/Tests/Tools.Tests.ps1 @@ -43,7 +43,9 @@ Describe "CMake" { Describe "CodeQLBundle" { It "CodeQLBundle" { - $CodeQLPath = Join-Path $Env:CODEQL_EXTRACTION_DIRECTORY -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" + $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "codeql" | Join-Path -ChildPath "*" + $CodeQLVersionPath = (Get-Item $CodeQLVersionsWildcard).FullPath + $CodeQLPath = Join-Path CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" "$CodeQLPath version" | Should -ReturnZeroExitCode } } From 53db4e15c431c12e04d168be293da25e6cbc808a Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Wed, 16 Sep 2020 14:11:22 +0100 Subject: [PATCH 07/20] Fix a typo. --- images/win/scripts/Tests/Tools.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/win/scripts/Tests/Tools.Tests.ps1 b/images/win/scripts/Tests/Tools.Tests.ps1 index 242ae296..027cc942 100644 --- a/images/win/scripts/Tests/Tools.Tests.ps1 +++ b/images/win/scripts/Tests/Tools.Tests.ps1 @@ -45,7 +45,7 @@ Describe "CodeQLBundle" { It "CodeQLBundle" { $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "codeql" | Join-Path -ChildPath "*" $CodeQLVersionPath = (Get-Item $CodeQLVersionsWildcard).FullPath - $CodeQLPath = Join-Path CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" + $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" "$CodeQLPath version" | Should -ReturnZeroExitCode } } From 6877c76a4e6aa1eab8b4583e346ab05fdc69e428 Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Thu, 17 Sep 2020 09:57:18 +0100 Subject: [PATCH 08/20] Fix another PowerShell issue. --- images/win/scripts/Tests/Tools.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/win/scripts/Tests/Tools.Tests.ps1 b/images/win/scripts/Tests/Tools.Tests.ps1 index 027cc942..cbb0609a 100644 --- a/images/win/scripts/Tests/Tools.Tests.ps1 +++ b/images/win/scripts/Tests/Tools.Tests.ps1 @@ -44,7 +44,7 @@ Describe "CMake" { Describe "CodeQLBundle" { It "CodeQLBundle" { $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "codeql" | Join-Path -ChildPath "*" - $CodeQLVersionPath = (Get-Item $CodeQLVersionsWildcard).FullPath + $CodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Select-Object -First 1 -Expand FullName $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" "$CodeQLPath version" | Should -ReturnZeroExitCode } From c72e08b8b6d226ae58eba7170344b3d81278d26d Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Thu, 17 Sep 2020 12:02:44 +0100 Subject: [PATCH 09/20] Touch a file to indicate the source of pre-downloaded CodeQL bundles. --- images/linux/scripts/installers/codeql-bundle.sh | 3 +++ images/win/scripts/Installers/Install-CodeQLBundle.ps1 | 3 +++ 2 files changed, 6 insertions(+) diff --git a/images/linux/scripts/installers/codeql-bundle.sh b/images/linux/scripts/installers/codeql-bundle.sh index 60008963..825ce070 100644 --- a/images/linux/scripts/installers/codeql-bundle.sh +++ b/images/linux/scripts/installers/codeql-bundle.sh @@ -18,5 +18,8 @@ echo "Downloading CodeQL bundle $codeql_bundle_version..." download_with_retries "https://github.com/github/codeql-action/releases/download/$codeql_bundle_name/codeql-bundle.tar.gz" "/tmp" "codeql-bundle.tar.gz" tar -xzf "/tmp/codeql-bundle.tar.gz" -C "$extraction_directory" +# Touch a special file that indicates to the CodeQL Action that this bundle was baked-in to the hosted runner images. +touch "$extraction_directory/pinned-version" + # Test that the tool has been extracted successfully. "$AGENT_TOOLSDIRECTORY/CodeQL/$codeql_bundle_version/x64/codeql/codeql" version diff --git a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 index 46a59759..d8f21f0d 100644 --- a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 +++ b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 @@ -18,5 +18,8 @@ Extract-7Zip -Path $CodeQLBundlePath -DestinationPath $DownloadDirectoryPath $UnGzipedCodeQLBundlePath = Join-Path $DownloadDirectoryPath "codeql-bundle.tar" Extract-7Zip -Path $UnGzipedCodeQLBundlePath -DestinationPath $ExtractionDirectory +# Touch a special file that indicates to the CodeQL Action that this bundle was baked-in to the hosted runner images. +New-Item -ItemType file (Join-Path $ExtractionDirectory -ChildPath "pinned-version") + # Test that the tool has been extracted successfully. Invoke-PesterTests -TestFile "Tools" -TestName "CodeQLBundle" From 60d40e5dd5ab9498790068244fbe52fe91c072be Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Thu, 17 Sep 2020 18:41:07 +0100 Subject: [PATCH 10/20] Document the CodeQL version that is installed. --- images/linux/scripts/installers/codeql-bundle.sh | 5 +++++ .../scripts/SoftwareReport/SoftwareReport.Generator.ps1 | 1 + .../win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 | 8 ++++++++ 3 files changed, 14 insertions(+) diff --git a/images/linux/scripts/installers/codeql-bundle.sh b/images/linux/scripts/installers/codeql-bundle.sh index 825ce070..e07f2a08 100644 --- a/images/linux/scripts/installers/codeql-bundle.sh +++ b/images/linux/scripts/installers/codeql-bundle.sh @@ -5,6 +5,7 @@ ################################################################################ source $HELPER_SCRIPTS/install.sh +source $HELPER_SCRIPTS/document.sh # Retrieve the name of the CodeQL bundle preferred by the Action (in the format codeql-bundle-YYYYMMDD). codeql_bundle_name="$(curl -sSL https://raw.githubusercontent.com/github/codeql-action/main/src/defaults.json | jq -r .bundleVersion)" @@ -23,3 +24,7 @@ touch "$extraction_directory/pinned-version" # Test that the tool has been extracted successfully. "$AGENT_TOOLSDIRECTORY/CodeQL/$codeql_bundle_version/x64/codeql/codeql" version + +# Document the version installed. +version="$("$AGENT_TOOLSDIRECTORY/CodeQL/$codeql_bundle_version/x64/codeql/codeql" version --quiet)" +DocumentInstalledItem "CodeQL Action Bundle ($version)" diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 index eb8b35f0..905b9da9 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 @@ -77,6 +77,7 @@ $markdown += New-MDList -Style Unordered -Lines @( (Get-BazelVersion), (Get-BazeliskVersion), (Get-CMakeVersion), + (Get-CodeQLBundleVersion), (Get-RVersion), (Get-DockerVersion), (Get-DockerComposeVersion), diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 b/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 index e7235408..e6d70fec 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 @@ -30,6 +30,14 @@ function Get-CMakeVersion { return "CMake $cmakeVersion" } +function Get-CodeQLBundleVersion { + $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "codeql" | Join-Path -ChildPath "*" + $CodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Select-Object -First 1 -Expand FullName + $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql.exe" + $CodeQLVersion = $($CodeQLPath version --quiet) + return "CodeQL Action Bundle $CodeQLVersion" +} + function Get-DockerVersion { $dockerVersion = $(docker version --format "{{.Server.Version}}") return "Docker $dockerVersion" From 49dc2ce2fd3a5f616ce8f7f5412ed918d928be14 Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Sun, 20 Sep 2020 09:29:43 +0100 Subject: [PATCH 11/20] Use new approach for documenting CodeQL on Ubuntu. --- .../scripts/SoftwareReport/SoftwareReport.Generator.ps1 | 1 + .../scripts/SoftwareReport/SoftwareReport.Tools.psm1 | 8 ++++++++ images/linux/scripts/installers/codeql-bundle.sh | 5 ----- .../win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 b/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 index 73a8d63e..7e6db374 100644 --- a/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 +++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 @@ -77,6 +77,7 @@ $toolsList = @( (Get-AzCopy10Version), (Get-BazelVersion), (Get-BazeliskVersion), + (Get-CodeQLBundleVersion), (Get-CMakeVersion), (Get-CurlVersion), (Get-DockerMobyVersion), diff --git a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 index dcb121f6..4f7d64e3 100644 --- a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 +++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 @@ -28,6 +28,14 @@ function Get-BazeliskVersion { return "Bazelisk $bazeliskVersion" } +function Get-CodeQLBundleVersion { + $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "codeql" | Join-Path -ChildPath "*" + $CodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Select-Object -First 1 -Expand FullName + $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" + $CodeQLVersion = $($CodeQLPath version --quiet) + return "CodeQL Action Bundle $CodeQLVersion" +} + function Get-PodManVersion { $podmanVersion = podman --version | Take-OutputPart -Part 2 return "Podman $podmanVersion" diff --git a/images/linux/scripts/installers/codeql-bundle.sh b/images/linux/scripts/installers/codeql-bundle.sh index e07f2a08..825ce070 100644 --- a/images/linux/scripts/installers/codeql-bundle.sh +++ b/images/linux/scripts/installers/codeql-bundle.sh @@ -5,7 +5,6 @@ ################################################################################ source $HELPER_SCRIPTS/install.sh -source $HELPER_SCRIPTS/document.sh # Retrieve the name of the CodeQL bundle preferred by the Action (in the format codeql-bundle-YYYYMMDD). codeql_bundle_name="$(curl -sSL https://raw.githubusercontent.com/github/codeql-action/main/src/defaults.json | jq -r .bundleVersion)" @@ -24,7 +23,3 @@ touch "$extraction_directory/pinned-version" # Test that the tool has been extracted successfully. "$AGENT_TOOLSDIRECTORY/CodeQL/$codeql_bundle_version/x64/codeql/codeql" version - -# Document the version installed. -version="$("$AGENT_TOOLSDIRECTORY/CodeQL/$codeql_bundle_version/x64/codeql/codeql" version --quiet)" -DocumentInstalledItem "CodeQL Action Bundle ($version)" diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 b/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 index e6d70fec..a90e13dd 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Tools.psm1 @@ -254,4 +254,4 @@ function Get-VisualCPPComponents { } } } -} \ No newline at end of file +} From 37668db1c4950a1e6c962ed6344db37bf405095f Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Mon, 21 Sep 2020 08:43:39 +0100 Subject: [PATCH 12/20] Fix CodeQL documentation function. --- images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 index 4f7d64e3..363b4adb 100644 --- a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 +++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 @@ -32,7 +32,7 @@ function Get-CodeQLBundleVersion { $CodeQLVersionsWildcard = Join-Path $Env:AGENT_TOOLSDIRECTORY -ChildPath "codeql" | Join-Path -ChildPath "*" $CodeQLVersionPath = Get-ChildItem $CodeQLVersionsWildcard | Select-Object -First 1 -Expand FullName $CodeQLPath = Join-Path $CodeQLVersionPath -ChildPath "x64" | Join-Path -ChildPath "codeql" | Join-Path -ChildPath "codeql" - $CodeQLVersion = $($CodeQLPath version --quiet) + $CodeQLVersion = & $CodeQLPath version --quiet return "CodeQL Action Bundle $CodeQLVersion" } From 0a8bab60939015b32e3f9b7c259aefb8d177d432 Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Mon, 21 Sep 2020 15:37:02 +0100 Subject: [PATCH 13/20] Touch a file required to indicate to the toolcache that CodeQL is fully set up. --- images/linux/scripts/installers/codeql-bundle.sh | 3 +++ images/win/scripts/Installers/Install-CodeQLBundle.ps1 | 3 +++ 2 files changed, 6 insertions(+) diff --git a/images/linux/scripts/installers/codeql-bundle.sh b/images/linux/scripts/installers/codeql-bundle.sh index 825ce070..9de8a5e4 100644 --- a/images/linux/scripts/installers/codeql-bundle.sh +++ b/images/linux/scripts/installers/codeql-bundle.sh @@ -21,5 +21,8 @@ tar -xzf "/tmp/codeql-bundle.tar.gz" -C "$extraction_directory" # Touch a special file that indicates to the CodeQL Action that this bundle was baked-in to the hosted runner images. touch "$extraction_directory/pinned-version" +# Touch a file to indicate to the toolcache that setting up CodeQL is complete. +touch "$extraction_directory.complete" + # Test that the tool has been extracted successfully. "$AGENT_TOOLSDIRECTORY/CodeQL/$codeql_bundle_version/x64/codeql/codeql" version diff --git a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 index d8f21f0d..d9f27377 100644 --- a/images/win/scripts/Installers/Install-CodeQLBundle.ps1 +++ b/images/win/scripts/Installers/Install-CodeQLBundle.ps1 @@ -21,5 +21,8 @@ Extract-7Zip -Path $UnGzipedCodeQLBundlePath -DestinationPath $ExtractionDirecto # Touch a special file that indicates to the CodeQL Action that this bundle was baked-in to the hosted runner images. New-Item -ItemType file (Join-Path $ExtractionDirectory -ChildPath "pinned-version") +# Touch a file to indicate to the toolcache that setting up CodeQL is complete. +New-Item -ItemType file "$ExtractionDirectory.complete" + # Test that the tool has been extracted successfully. Invoke-PesterTests -TestFile "Tools" -TestName "CodeQLBundle" From 083046a33cd6c680382c237c02ce81c934836154 Mon Sep 17 00:00:00 2001 From: Darii Nurgaleev Date: Thu, 24 Sep 2020 19:05:39 +0700 Subject: [PATCH 14/20] added toolsets --- images/macos/toolsets/toolset-10.15.json | 4 ++-- images/macos/toolsets/toolset-11.0.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/images/macos/toolsets/toolset-10.15.json b/images/macos/toolsets/toolset-10.15.json index e7b17989..26c0cf19 100644 --- a/images/macos/toolsets/toolset-10.15.json +++ b/images/macos/toolsets/toolset-10.15.json @@ -6,12 +6,12 @@ ] }, "xamarin": { - "vsmac": "8.7.7.10", + "vsmac": "8.7.8.4", "mono-versions": [ "6.12.0.93", "6.10.0.106", "6.8.0.123", "6.6.0.166", "6.4.0.208" ], "ios-versions": [ - "13.20.2.2", "13.18.2.1", "13.16.0.13", "13.14.1.39", "13.10.0.21", "13.8.3.0", "13.6.0.12", "13.4.0.2", "13.2.0.47" + "14.0.0.0", "13.20.2.2", "13.18.2.1", "13.16.0.13", "13.14.1.39", "13.10.0.21", "13.8.3.0", "13.6.0.12", "13.4.0.2", "13.2.0.47" ], "mac-versions": [ "6.20.2.2", "6.18.3.2", "6.16.0.13", "6.14.1.39", "6.10.0.21", "6.8.3.0", "6.6.0.12", "6.4.0.2", "6.2.0.47" diff --git a/images/macos/toolsets/toolset-11.0.json b/images/macos/toolsets/toolset-11.0.json index fa7a65a5..3f139910 100644 --- a/images/macos/toolsets/toolset-11.0.json +++ b/images/macos/toolsets/toolset-11.0.json @@ -6,12 +6,12 @@ ] }, "xamarin": { - "vsmac": "8.7.7.10", + "vsmac": "8.7.8.4", "mono-versions": [ "6.12.0.93" ], "ios-versions": [ - "13.20.2.2" + "14.0.0.0", "13.20.2.2" ], "mac-versions": [ "6.20.2.2" From ffbcaa7854620e2d260de98869b05862550de5c2 Mon Sep 17 00:00:00 2001 From: Dibir Magomedsaygitov Date: Thu, 24 Sep 2020 15:07:30 +0300 Subject: [PATCH 15/20] add missing tools to ubuntu docs --- .../SoftwareReport/SoftwareReport.Generator.ps1 | 4 ++++ .../scripts/SoftwareReport/SoftwareReport.Tools.psm1 | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 b/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 index 73a8d63e..5c93cdd5 100644 --- a/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 +++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 @@ -101,7 +101,11 @@ $toolsList = @( (Get-NewmanVersion), (Get-NvmVersion), (Get-PackerVersion), + ("Pollinate"), (Get-PhantomJSVersion), + (Get-PulumiVersion), + (Get-RVersion), + ("Sphinx Open Source Search Server"), (Get-SwigVersion), (Get-TerraformVersion), (Get-UnZipVersion), diff --git a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 index dcb121f6..43828d48 100644 --- a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 +++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 @@ -247,3 +247,13 @@ function Get-ORASCliVersion { function Get-VerselCliversion { return "$(vercel --version 2>&1 | Select-Object -First 1)" } + +function Get-PulumiVersion { + $pulumiVersion = pulumi version | Take-OutputPart -Part 0 -Delimiter "v" + return "Pulumi $pulumiVersion" +} + +function Get-RVersion { + $rVersion = R --version | Select-String "R version" | Take-OutputPart -Part -2 + return "R $rVersion" +} From 3500bbdf3e471b887d4e4b99aecefd79d9d61d80 Mon Sep 17 00:00:00 2001 From: Dibir Magomedsaygitov Date: Thu, 24 Sep 2020 16:50:28 +0300 Subject: [PATCH 16/20] add sphinx version and remove pollinate --- .../scripts/SoftwareReport/SoftwareReport.Generator.ps1 | 3 +-- .../linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 b/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 index 5c93cdd5..f4dce82c 100644 --- a/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 +++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 @@ -101,11 +101,10 @@ $toolsList = @( (Get-NewmanVersion), (Get-NvmVersion), (Get-PackerVersion), - ("Pollinate"), (Get-PhantomJSVersion), (Get-PulumiVersion), (Get-RVersion), - ("Sphinx Open Source Search Server"), + (Get-SphinxVersion), (Get-SwigVersion), (Get-TerraformVersion), (Get-UnZipVersion), diff --git a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 index 43828d48..35b5ec4d 100644 --- a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 +++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 @@ -257,3 +257,8 @@ function Get-RVersion { $rVersion = R --version | Select-String "R version" | Take-OutputPart -Part -2 return "R $rVersion" } + +function Get-SphinxVersion { + $sphinxVersion = searchd | Take-OutputPart -Part 1 | Take-OutputPart -Part 0 -Delimiter "-" + return "Sphinx Open Source Search Server $rVersion" +} \ No newline at end of file From 9cc2c6a139e1babe44f0ccebca0efe4bc73add51 Mon Sep 17 00:00:00 2001 From: Darii Nurgaleev Date: Thu, 24 Sep 2020 22:24:23 +0700 Subject: [PATCH 17/20] added bundles --- images/macos/toolsets/toolset-10.15.json | 7 +++++++ images/macos/toolsets/toolset-11.0.json | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/images/macos/toolsets/toolset-10.15.json b/images/macos/toolsets/toolset-10.15.json index 26c0cf19..587da961 100644 --- a/images/macos/toolsets/toolset-10.15.json +++ b/images/macos/toolsets/toolset-10.15.json @@ -21,6 +21,13 @@ ], "bundle-default": "latest", "bundles": [ + { + "symlink": "6_12_1", + "mono":"6.12", + "ios": "14.0", + "mac": "6.20", + "android": "11.0" + }, { "symlink": "6_12_0", "mono":"6.12", diff --git a/images/macos/toolsets/toolset-11.0.json b/images/macos/toolsets/toolset-11.0.json index 3f139910..c8988a4b 100644 --- a/images/macos/toolsets/toolset-11.0.json +++ b/images/macos/toolsets/toolset-11.0.json @@ -21,6 +21,13 @@ ], "bundle-default": "latest", "bundles": [ + { + "symlink": "6_12_1", + "mono":"6.12", + "ios": "14.0", + "mac": "6.20", + "android": "11.0" + }, { "symlink": "6_12_0", "mono":"6.12", From 85906da062ec55b2ab9d0f1070fc0df0be114f3c Mon Sep 17 00:00:00 2001 From: Dibir Magomedsaygitov Date: Thu, 24 Sep 2020 23:39:14 +0300 Subject: [PATCH 18/20] fix R and sphinx version output --- .../linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 index 35b5ec4d..99fad98b 100644 --- a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 +++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1 @@ -254,11 +254,11 @@ function Get-PulumiVersion { } function Get-RVersion { - $rVersion = R --version | Select-String "R version" | Take-OutputPart -Part -2 + $rVersion = (Get-CommandResult "R --version | grep 'R version'").Output | Take-OutputPart -Part 2 return "R $rVersion" } function Get-SphinxVersion { - $sphinxVersion = searchd | Take-OutputPart -Part 1 | Take-OutputPart -Part 0 -Delimiter "-" - return "Sphinx Open Source Search Server $rVersion" + $sphinxVersion = searchd -h | Select-Object -First 1 | Take-OutputPart -Part 1 | Take-OutputPart -Part 0 -Delimiter "-" + return "Sphinx Open Source Search Server $sphinxVersion" } \ No newline at end of file From 3ffe5b2edcd856a68b35fad13b872437cfea67b5 Mon Sep 17 00:00:00 2001 From: Darii Nurgaleev Date: Fri, 25 Sep 2020 12:09:37 +0700 Subject: [PATCH 19/20] xcode 12.0.1 --- images/macos/toolsets/toolset-10.15.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/macos/toolsets/toolset-10.15.json b/images/macos/toolsets/toolset-10.15.json index 587da961..e4c92cce 100644 --- a/images/macos/toolsets/toolset-10.15.json +++ b/images/macos/toolsets/toolset-10.15.json @@ -2,7 +2,7 @@ "xcode": { "default": "11.7", "versions": [ - "12.2_beta", "12", "12_beta", "11.7", "11.6", "11.5", "11.4.1", "11.4", "11.3.1", "11.2.1", "11.1", "11", "10.3" + "12.2_beta", "12.0.1", "12", "12_beta", "11.7", "11.6", "11.5", "11.4.1", "11.4", "11.3.1", "11.2.1", "11.1", "11", "10.3" ] }, "xamarin": { From 2f4d3598593f05840832ac695bfc421c51424671 Mon Sep 17 00:00:00 2001 From: Darii Nurgaleev Date: Fri, 25 Sep 2020 19:08:50 +0700 Subject: [PATCH 20/20] removed 12.0.1 --- images/macos/toolsets/toolset-10.15.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/macos/toolsets/toolset-10.15.json b/images/macos/toolsets/toolset-10.15.json index e4c92cce..587da961 100644 --- a/images/macos/toolsets/toolset-10.15.json +++ b/images/macos/toolsets/toolset-10.15.json @@ -2,7 +2,7 @@ "xcode": { "default": "11.7", "versions": [ - "12.2_beta", "12.0.1", "12", "12_beta", "11.7", "11.6", "11.5", "11.4.1", "11.4", "11.3.1", "11.2.1", "11.1", "11", "10.3" + "12.2_beta", "12", "12_beta", "11.7", "11.6", "11.5", "11.4.1", "11.4", "11.3.1", "11.2.1", "11.1", "11", "10.3" ] }, "xamarin": {