Switch Windows hostedtoolcache provisioner to GitHub Actions NPM registry (#315)

* Toolcache: add basic error handling in Windows toolcache provisioner

* Toolcache: add GITHUB_FEED_TOKEN into packer templates for Windows

* Toolcache: separate toolcache.json for Windows

* Rework "Validate-ToolCache" script (#6)

* Rework validate toolcache script

* change boost root folder

* Add boost 1.69 for windows-2019

* Add toolcache config variable to Windows 16, 19

* Revert "Add toolcache config variable to Windows 16, 19"

* Add default boost version for validate-boost VS19

* Remove bjam test for boost

* Add boost 1.72 validation for win-2016

Co-authored-by: MaksimZhukov <46996400+MaksimZhukov@users.noreply.github.com>
Co-authored-by: Aleksandr Chebotov <47745270+al-cheb@users.noreply.github.com>

images/win/Windows2016-Azure.json

@@ -26,9 +26,9 @@
         "install_password": null,
         "capture_name_prefix": "packer",
         "image_version": "dev",
-        "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/"
+        "github_feed_token": "{{env `GITHUB_FEED_TOKEN`}}"
     },
-    "sensitive-variables": ["install_password", "ssh_password", "client_secret"],
+    "sensitive-variables": ["install_password", "ssh_password", "client_secret", "github_feed_token"],
     "builders": [
         {
             "name": "vhd",
@@ -283,13 +283,13 @@
         },
         {
             "type": "file",
-            "source": "{{template_dir}}/toolcache.json",
+            "source": "{{template_dir}}/toolcache-2016.json",
             "destination": "{{user `root_folder`}}/toolcache.json"
         },
         {
             "type": "powershell",
             "environment_vars":[
-                "TOOLCACHE_REGISTRY={{ user `toolcache_registry` }}",
+                "GITHUB_FEED_TOKEN={{ user `github_feed_token` }}",
                 "ROOT_FOLDER={{user `root_folder`}}"
             ],
             "scripts":[
@@ -477,7 +477,7 @@
         {
             "type": "powershell",
             "environment_vars": [
-                "BOOST_VERSIONS=1.69.0",
+                "BOOST_VERSIONS=1.69.0,1.72.0",
                 "BOOST_DEFAULT=1.69.0"
             ],
             "scripts":[
@@ -612,7 +612,7 @@
         {
             "type": "powershell",
             "environment_vars": [
-                "BOOST_VERSIONS=1.69.0",
+                "BOOST_VERSIONS=1.69.0,1.72.0",
                 "BOOST_DEFAULT=1.69.0"
             ],
             "scripts":[

images/win/Windows2019-Azure.json

@@ -26,9 +26,9 @@
         "install_password": null,
         "capture_name_prefix": "packer",
         "image_version": "dev",
-        "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/"
+        "github_feed_token": "{{env `GITHUB_FEED_TOKEN`}}"
     },
-    "sensitive-variables": ["install_password", "ssh_password", "client_secret"],
+    "sensitive-variables": ["install_password", "ssh_password", "client_secret", "github_feed_token"],
     "builders": [
         {
             "name": "vhd",
@@ -252,13 +252,13 @@
         },
         {
             "type": "file",
-            "source": "{{template_dir}}/toolcache.json",
+            "source": "{{template_dir}}/toolcache-2019.json",
             "destination": "{{user `root_folder`}}/toolcache.json"
         },
         {
             "type": "powershell",
             "environment_vars":[
-                "TOOLCACHE_REGISTRY={{ user `toolcache_registry` }}",
+                "GITHUB_FEED_TOKEN={{ user `github_feed_token` }}",
                 "ROOT_FOLDER={{user `root_folder`}}"
             ],
             "scripts":[
@@ -446,8 +446,8 @@
         {
             "type": "powershell",
             "environment_vars": [
-                "BOOST_VERSIONS=1.69.0",
-                "BOOST_DEFAULT=1.69.0"
+                "BOOST_VERSIONS=1.69.0,1.72.0",
+                "BOOST_DEFAULT=1.72.0"
             ],
             "scripts":[
                 "{{ template_dir }}/scripts/Installers/Install-Boost.ps1"
@@ -581,8 +581,8 @@
         {
             "type": "powershell",
             "environment_vars": [
-                "BOOST_VERSIONS=1.69.0",
-                "BOOST_DEFAULT=1.69.0"
+                "BOOST_VERSIONS=1.69.0,1.72.0",
+                "BOOST_DEFAULT=1.72.0"
             ],
             "scripts":[
                 "{{ template_dir }}/scripts/Installers/Validate-Boost.ps1"

images/win/scripts/Installers/Download-ToolCache.ps1

@@ -8,18 +8,45 @@ Function Install-NpmPackage {
     [CmdletBinding()]
     param(
         [Parameter(Mandatory=$true)]
-        [System.String]
-        $Name,
+        [System.String] $PackageName,
         [Parameter(Mandatory=$true)]
-        [System.String]
-        $NpmRegistry
+        [System.Uri] $FeedPrefix
     )
 
-    Write-Host "Installing npm '$Name' package from '$NpmRegistry'"
+    Push-Location -Path $env:TEMP
 
-    npm install $Name --registry=$NpmRegistry
+    $FeedUri = $FeedPrefix.AbsoluteUri
+
+    Write-Host "Installing npm $PackageName from ${FeedUri}"
+    npm install $PackageName --registry "${FeedUri}"
+
+    if($LASTEXITCODE) {
+        Write-Host "$PackageName installation failure;  Error: ${LASTEXITCODE}"
+
+        exit 1
+    }
+
+    Pop-Location
 }
 
+Function NPMFeed-AuthSetup {
+    param(
+        [Parameter(Mandatory=$true)]
+        [System.String] $AccessToken,
+        [Parameter(Mandatory=$true)]
+        [System.Uri] $FeedPrefix
+    )
+
+    $FeedHost = $FeedPrefix.Host
+
+    Write-Host "Configure auth for github package registry"
+    $npmrcContent = "//${FeedHost}/:_authToken=${AccessToken}"
+    $npmrcContent | Out-File -FilePath "$($env:TEMP)/.npmrc" -Encoding utf8
+}
+
+$FeedPrefix = "https://npm.pkg.github.com"
+$AccessToken = $env:GITHUB_FEED_TOKEN
+
 # HostedToolCache Path
 $Dest = "C:/"
 $Path = "hostedtoolcache/windows"
@@ -33,16 +60,18 @@ setx AGENT_TOOLSDIRECTORY $ToolsDirectory /M
 $ToolVersionsFileContent = Get-Content -Path "$env:ROOT_FOLDER/toolcache.json" -Raw
 $ToolVersions = ConvertFrom-Json -InputObject $ToolVersionsFileContent
 
+NPMFeed-AuthSetup -AccessToken $AccessToken -FeedPrefix $FeedPrefix
+
 $ToolVersions.PSObject.Properties | ForEach-Object {
     $PackageName = $_.Name
     $PackageVersions = $_.Value
     $NpmPackages = $PackageVersions | ForEach-Object { "$PackageName@$_" }
     foreach($NpmPackage in $NpmPackages) {
-        Install-NpmPackage -Name $NpmPackage -NpmRegistry $env:TOOLCACHE_REGISTRY
+        Install-NpmPackage -PackageName $NpmPackage -FeedPrefix $FeedPrefix
     }
 }
 
 #junction point from the previous Python2 directory to the toolcache Python2
 Write-Host "Create symlink to Python2"
 $python2Dir = (Get-Item -Path ($ToolsDirectory + '/Python/2.7*/x64')).FullName
-cmd.exe /c mklink /d "C:\Python27amd64" "$python2Dir"
+cmd.exe /c mklink /d "C:\Python27amd64" "$python2Dir"

images/win/scripts/Installers/Install-Boost.ps1

@@ -4,7 +4,7 @@
 ##  Desc:  Install boost using tool cache
 ################################################################################
 
-$BoostDirectory = Join-Path -Path $env:ProgramFiles -ChildPath "Boost"
+$BoostDirectory = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "Boost"
 $BoostVersions = $env:BOOST_VERSIONS.split(',')
 $BoostDefault = $env:BOOST_DEFAULT
 

images/win/scripts/Installers/Validate-Boost.ps1

@@ -13,10 +13,10 @@ function Validate-BoostVersion
 
     $ReleasePath = Join-Path -Path $BoostRootPath -ChildPath $BoostRelease
 
-    if ((Test-Path "$ReleasePath\b2.exe") -and (Test-Path "$ReleasePath\bjam.exe"))
+    if (Test-Path "$ReleasePath\b2.exe")
     {
         Write-Host "Boost.Build $BoostRelease is successfully installed"
-        Write-Host "Boost.Jam $BoostRelease is successfully installed"
+
         return
     }
 
@@ -25,7 +25,7 @@ function Validate-BoostVersion
 }
 
 # Verify that Boost is on the path
-if ((Get-Command -Name 'b2') -and (Get-Command -Name 'bjam'))
+if (Get-Command -Name 'b2')
 {
     Write-Host "Boost is on the path"
 }
@@ -54,15 +54,15 @@ $tmplMarkRoot = @"
 
 $SoftwareName = 'Boost'
 $Description = New-Object System.Text.StringBuilder
-$BoostRootDirectory = Join-Path -Path $env:ProgramFiles -ChildPath "Boost"
+$BoostRootDirectory = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "Boost"
 $BoostVersionsToInstall = $env:BOOST_VERSIONS.split(",")
 
-foreach($Boost in $BoostVersionsToInstall)
+foreach($BoostVersion in $BoostVersionsToInstall)
 {
-    Validate-BoostVersion -BoostRootPath $BoostRootDirectory -BoostRelease $Boost
-    $BoostVersionTag = "BOOST_ROOT_{0}" -f $Boost.Replace('.', '_')
+    Validate-BoostVersion -BoostRootPath $BoostRootDirectory -BoostRelease $BoostVersion
+    $BoostVersionTag = "BOOST_ROOT_{0}" -f $BoostVersion.Replace('.', '_')
 
-    if($boost -eq $env:BOOST_DEFAULT)
+    if($BoostVersion -eq $env:BOOST_DEFAULT)
     {
         $null = $Description.AppendLine(($tmplMarkRoot -f $BoostVersion, $BoostVersionTag))
     }

images/win/scripts/Installers/Validate-ToolCache.ps1

@@ -17,16 +17,62 @@ function Get-ToolcachePackages {
     return Get-Content -Raw $toolcachePath | ConvertFrom-Json
 }
 
-$packages = (Get-ToolcachePackages).PSObject.Properties | ForEach-Object {
+$toolcachePackages = (Get-ToolcachePackages).PSObject.Properties | ForEach-Object {
     $packageNameParts = $_.Name.Split("-")
-    $toolName = $packageNameParts[1]
     return [PSCustomObject] @{
         ToolName = $packageNameParts[1]
         Versions = $_.Value
-        Arch = $packageNameParts[3]
+        Architecture = $packageNameParts[3]
     }
 }
 
+function GetToolsByName {
+    param (
+        [Parameter(Mandatory = $True)]
+        [string]$SoftwareName
+    )
+    return $toolcachePackages | Where-Object { $_.ToolName -eq $SoftwareName }
+}
+
+function RunTestsByPath {
+    param (
+        [Parameter(Mandatory = $True)]
+        [string[]]$ExecTests,
+        [Parameter(Mandatory = $True)]
+        [string]$Path,
+        [Parameter(Mandatory = $True)]
+        [string]$SoftwareName,
+        [Parameter(Mandatory = $True)]
+        [string]$SoftwareVersion,
+        [Parameter(Mandatory = $True)]
+        [string]$SoftwareArchitecture
+    )
+
+    foreach ($test in $ExecTests)
+    {
+        if (Test-Path "$Path\$test")
+        {
+            Write-Host "$SoftwareName($test) $SoftwareVersion($SoftwareArchitecture) is successfully installed:"
+            Write-Host (& "$Path\$test" --version)
+        }
+        else
+        {
+            Write-Host "$SoftwareName($test) $SoftwareVersion($SoftwareArchitecture) is not installed"
+            exit 1
+        }
+    }
+}
+
+function GetMarkdownDescription {
+    param (
+        [Parameter(Mandatory = $True)]
+        [string]$SoftwareVersion,
+        [Parameter(Mandatory = $True)]
+        [string]$SoftwareArchitecture
+    )
+    return "_Version:_ $SoftwareVersion ($SoftwareArchitecture)<br/>"
+}
+
 function ToolcacheTest {
     param (
         [Parameter(Mandatory = $True)]
@@ -34,68 +80,50 @@ function ToolcacheTest {
         [Parameter(Mandatory = $True)]
         [string[]]$ExecTests
     )
-    if (Test-Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName")
+
+    $softwarePath = "$env:AGENT_TOOLSDIRECTORY\$SoftwareName"
+
+    if (-Not (Test-Path $softwarePath))
     {
-        $description = ""
-        [array]$installedVersions = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName"
-        if ($installedVersions.count -gt 0){
-            $softwarePackages = $packages | Where-Object { $_.ToolName -eq $SoftwareName }
-            foreach($softwarePackage in $softwarePackages)
-            {
-                foreach ($version in $softwarePackage.Versions)
-                {
-                    $foundVersion = $installedVersions | where { $_.StartsWith($version) }
-
-                    if ($foundVersion -ne $null){
-
-                        $architectures = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$foundVersion"
-
-                        $softwareArch = $softwarePackage.Arch
-
-                        if ($architectures -Contains $softwareArch) {
-                            $path = "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$foundVersion\$softwareArch"
-                            foreach ($test in $ExecTests)
-                            {
-                                if (Test-Path "$path\$test")
-                                {
-                                    Write-Host "$SoftwareName($test) $foundVersion($softwareArch) is successfully installed:"
-                                    Write-Host (& "$path\$test" --version)
-                                }
-                                else
-                                {
-                                    Write-Host "$SoftwareName($test) $foundVersion ($softwareArch) is not installed"
-                                    exit 1
-                                }
-                            }
-                            $description += "_Version:_ $foundVersion ($softwareArch)<br/>"
-                        }
-                        else
-                        {
-                            Write-Host "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$foundVersion does not include required architecture"
-                            exit 1
-                        }
-                    }
-                    else
-                    {
-                        Write-Host "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$version.* was not found"
-                        exit 1
-                    }
-                }
-
-                Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $description
-            }
-        }
-        else
-        {
-            Write-Host "$env:AGENT_TOOLSDIRECTORY\$SoftwareName does not include any folders"
-            exit 1
-        }
-    }
-    else
-    {
-        Write-Host "$env:AGENT_TOOLSDIRECTORY\$SoftwareName does not exist"
+        Write-Host "$softwarePath does not exist"
         exit 1
     }
+
+    [array]$installedVersions = GetChildFolders -Path $softwarePath
+    if ($installedVersions.count -eq 0)
+    {
+        Write-Host "$softwarePath does not include any folders"
+        exit 1
+    }
+
+    $markdownDescription = ""
+    $tools = GetToolsByName -SoftwareName $SoftwareName
+    foreach($tool in $tools)
+    {
+        foreach ($version in $tool.Versions)
+        {
+            $foundVersion = $installedVersions | where { $_.StartsWith($version) }
+            if ($foundVersion -eq $null)
+            {
+                Write-Host "$softwarePath\$version.* was not found"
+                exit 1
+            }
+
+            $installedArchitecture = GetChildFolders -Path "$softwarePath\$foundVersion"
+            $requiredArchitecture = $tool.Architecture
+            if (-Not ($installedArchitecture -Contains $requiredArchitecture))
+            {
+                Write-Host "$softwarePath\$foundVersion does not include the $requiredArchitecture architecture"
+                exit 1
+            }
+
+            $path = "$softwarePath\$foundVersion\$requiredArchitecture"
+            RunTestsByPath -ExecTests $ExecTests -Path $path -SoftwareName $SoftwareName -SoftwareVersion $foundVersion -SoftwareArchitecture $requiredArchitecture
+
+            $markdownDescription += GetMarkdownDescription -SoftwareVersion $foundVersion -SoftwareArchitecture $requiredArchitecture
+        }
+    }
+    Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $markdownDescription
 }
 
 # Python test

images/win/toolcache-2016.json

@@ -0,0 +1,17 @@
+{
+    "@actions/toolcache-python-windows-x64": [
+        "2.7", "3.5", "3.6", "3.7", "3.8"
+    ],
+    "@actions/toolcache-python-windows-x86": [
+        "2.7", "3.5", "3.6", "3.7", "3.8"
+    ],
+    "@actions/toolcache-ruby-windows-x64": [
+        "2.4", "2.5", "2.6", "2.7"
+    ],
+    "@actions/toolcache-pypy-windows-x86": [
+        "2", "3"
+    ],
+    "@actions/toolcache-boost-windows-msvc-14.1-x32-x64": [
+        "1.69", "1.72"
+    ]
+}

images/win/toolcache-2019.json

@@ -0,0 +1,20 @@
+{
+    "@actions/toolcache-python-windows-x64": [
+        "2.7", "3.5", "3.6", "3.7", "3.8"
+    ],
+    "@actions/toolcache-python-windows-x86": [
+        "2.7", "3.5", "3.6", "3.7", "3.8"
+    ],
+    "@actions/toolcache-ruby-windows-x64": [
+        "2.4", "2.5", "2.6", "2.7"
+    ],
+    "@actions/toolcache-pypy-windows-x86": [
+        "2", "3"
+    ],
+    "@actions/toolcache-boost-windows-msvc-14.1-x32-x64": [
+        "1.69"
+    ],
+    "@actions/toolcache-boost-windows-msvc-14.2-x32-x64": [
+        "1.72"
+    ]
+}

images/win/toolcache.json

@@ -1,17 +0,0 @@
-{
-    "toolcache-python-windows-x64": [
-        "2.7", "3.5", "3.6", "3.7", "3.8"
-    ],
-    "toolcache-python-windows-x86": [
-        "2.7", "3.5", "3.6", "3.7", "3.8"
-    ],
-    "toolcache-ruby-windows-x64": [
-        "2.4", "2.5", "2.6", "2.7"
-    ],
-    "toolcache-pypy-windows-x86": [
-        "2", "3"
-    ],
-    "toolcache-boost-windows-x64": [
-        "1.69"
-    ]
-}