Switch provisioners to install Go from GitHub releases on Windows (#1026)

* add go to json toolset

* add install and validate go

* fiixing go tests

* fix validation for go

* check zero build version

* add 1.9 version

* setup variables for go

* fix comments

* remove script invokation

* remove lines

* fixing install go

* remove comments

* remove fake url

Co-authored-by: Dmitry Shibanov <v-dmshib@microsoft.com>

images/win/Windows2016-Azure.json

@@ -30,9 +30,7 @@
         "capture_name_prefix": "packer",
         "image_version": "dev",
         "image_os": "win16",
-        "github_feed_token": "{{env `GITHUB_FEED_TOKEN`}}",
+        "github_feed_token": "{{env `GITHUB_FEED_TOKEN`}}"
-        "go_versions": "1.9, 1.10, 1.11, 1.12, 1.13, 1.14",
-        "go_default": "1.14"
     },
     "sensitive-variables": ["install_password", "ssh_password", "client_secret", "github_feed_token"],
     "builders": [
@@ -375,6 +373,15 @@
                 "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1"
             ]
         },
+        {
+            "type": "powershell",
+            "environment_vars":[
+                "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
+            ],
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Install-Go.ps1"
+            ]
+        },
         {
             "type": "powershell",
             "scripts":[
@@ -399,16 +406,6 @@
                 "{{ template_dir }}/scripts/Installers/Install-GitHub-CLI.ps1"
             ]
         },
-        {
-            "type": "powershell",
-            "environment_vars": [
-                "GO_VERSIONS={{user `go_versions`}}",
-                "GO_DEFAULT={{user `go_default`}}"
-            ],
-            "scripts":[
-                "{{ template_dir }}/scripts/Installers/Install-Go.ps1"
-            ]
-        },
         {
             "type": "powershell",
             "scripts":[
@@ -779,16 +776,6 @@
                 "{{ template_dir }}/scripts/Installers/Validate-GitHub-CLI.ps1"
             ]
         },
-        {
-            "type": "powershell",
-            "environment_vars": [
-                "GO_VERSIONS={{user `go_versions`}}",
-                "GO_DEFAULT={{ user `go_default`}}"
-            ],
-            "scripts":[
-                "{{ template_dir }}/scripts/Installers/Validate-Go.ps1"
-            ]
-        },
         {
             "type": "powershell",
             "scripts":[

images/win/Windows2019-Azure.json

@@ -30,9 +30,7 @@
         "capture_name_prefix": "packer",
         "image_version": "dev",
         "image_os": "win19",
-        "github_feed_token": "{{env `GITHUB_FEED_TOKEN`}}",
+        "github_feed_token": "{{env `GITHUB_FEED_TOKEN`}}"
-        "go_versions": "1.9, 1.10, 1.11, 1.12, 1.13, 1.14",
-        "go_default": "1.14"
     },
     "sensitive-variables": ["install_password", "ssh_password", "client_secret", "github_feed_token"],
     "builders": [
@@ -342,6 +340,15 @@
                 "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1"
             ]
         },
+        {
+            "type": "powershell",
+            "environment_vars":[
+                "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
+            ],
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Install-Go.ps1"
+            ]
+        },
         {
             "type": "powershell",
             "scripts":[
@@ -365,16 +372,6 @@
             "scripts":[
                 "{{ template_dir }}/scripts/Installers/Install-GitHub-CLI.ps1"
             ]
-        },
-        {
-            "type": "powershell",
-            "environment_vars": [
-                "GO_VERSIONS={{user `go_versions`}}",
-                "GO_DEFAULT={{user `go_default`}}"
-            ],
-            "scripts":[
-                "{{ template_dir }}/scripts/Installers/Install-Go.ps1"
-            ]
         },
          {
             "type": "powershell",
@@ -764,16 +761,6 @@
                 "{{ template_dir }}/scripts/Installers/Validate-GitHub-CLI.ps1"
             ]
         },
-        {
-            "type": "powershell",
-            "environment_vars": [
-                "GO_VERSIONS={{user `go_versions`}}",
-                "GO_DEFAULT={{user `go_default`}}"
-            ],
-            "scripts":[
-                "{{ template_dir }}/scripts/Installers/Validate-Go.ps1"
-            ]
-        },
         {
             "type": "powershell",
             "scripts":[

images/win/scripts/Installers/Install-Go.ps1

@@ -3,83 +3,23 @@
 ##  Desc:  Install Go
 ################################################################################
 
-Import-Module -Name ImageHelpers -Force
+Import-Module -Name ImageHelpers -Force -DisableNameChecking
 
-$refsJson = Invoke-RestMethod "https://api.github.com/repos/golang/go/git/refs/tags"
+# Get Go content from toolset
+$goTool = Get-ToolsetContent | Select-Object -ExpandProperty toolcache | Where-Object Name -eq "go"
 
-function Install-GoVersion
+$toolPath = Join-Path $env:AGENT_TOOLSDIRECTORY $goTool.name
+foreach($goVersion in $goTool.versions)
 {
-    Param
+    if ($goVersion.Split(".").Length -lt 3) {
-    (
+        $goVersion += ".*"
-        [String] $goVersion,
-        [Switch] $addToDefaultPath
-    )
-
-    $latestVersionObject = $refsJson | Where-Object { $_.ref -Match "refs/tags/go$goVersion[./d]*" } | Select-Object -Last 1
-    $latestVersion = $latestVersionObject.ref.replace('refs/tags/go','')
-
-    # Download the Go zip archive.
-    Write-Host "Downloading Go $latestVersion..."
-    $ProgressPreference = 'SilentlyContinue'
-
-    $goArchName = "go${latestVersion}.windows-amd64.zip"
-    $goArchUrl = "https://dl.google.com/go/${goArchName}"
-
-    $goArchPath = Start-DownloadWithRetry -Url $goArchUrl -Name $goArchName
-
-    # Extract the zip archive.  It contains a single directory named "go".
-    Write-Host "Extracting Go $latestVersion..."
-    $toolDirectory = Join-Path $env:AGENT_TOOLSDIRECTORY "go\$latestVersion"
-    Extract-7Zip -Path $goArchPath -DestinationPath $toolDirectory
-
-    # Rename the extracted "go" directory to "x64" for full path "C:\hostedtoolcache\windows\Go\1.14.2\x64\..."
-    Rename-Item -path "$toolDirectory\go" -newName "x64"
-    $fullArchPath = "$toolDirectory\x64"
-
-    # Delete unnecessary files to conserve space
-    Write-Host "Cleaning directories of Go $latestVersion..."
-    if (Test-Path "$fullArchPath\doc")
-    {
-        Remove-Item -Recurse -Force "$fullArchPath\doc"
     }
-    if (Test-Path "$fullArchPath\blog")
+    $expectedVersionPath = Join-Path $toolPath $goVersion
-    {
+    $foundVersion = Get-Item $expectedVersionPath `
-        Remove-Item -Recurse -Force "$fullArchPath\blog"
+                    | Sort-Object -Property {[version]$_.name} -Descending `
-    }
+                    | Select-Object -First 1
-
+    # Check for required architecture folder
-    # Create symlink in old location
+    $foundVersionArchPath = Join-Path $foundVersion $goTool.arch
-    New-Item -Path "C:\go$latestVersion" -ItemType SymbolicLink -Value $fullArchPath
+    $envName = "GOROOT_{0}_{1}_X64" -f $goVersion.split(".")
-
+    setx $envName "$foundVersionArchPath" /M
-    # Make this the default version of Go?
+}
-    if ($addToDefaultPath)
-    {
-        Write-Host "Adding Go $latestVersion to the path..."
-        # Add the Go binaries to the path.
-        Add-MachinePathItem "$fullArchPath\bin" | Out-Null
-        # Set the GOROOT environment variable.
-        setx GOROOT "$fullArchPath" /M | Out-Null
-    }
-
-    # Done
-    Write-Host "Done installing Go $latestVersion."
-    return $fullArchPath
-}
-
-# Install Go
-$goVersionsToInstall = $env:GO_VERSIONS.split(", ", [System.StringSplitOptions]::RemoveEmptyEntries)
-
-foreach ($go in $goVersionsToInstall)
-{
-    Write-Host "Installing Go ${go}"
-    if ($go -eq $env:GO_DEFAULT)
-    {
-        $installDirectory = Install-GoVersion -goVersion $go -addToDefaultPath
-    }
-    else
-    {
-        $installDirectory = Install-GoVersion -goVersion $go
-    }
-
-    $envName = "GOROOT_{0}_{1}_X64" -f $go.split(".")
-    setx $envName "$installDirectory" /M
-}

images/win/scripts/Installers/Install-Toolset.ps1

@@ -49,12 +49,34 @@ Function Set-DefaultPythonVersion {
     }
 }
 
+Function Set-DefaultGoVersion {
+    param(
+        [Parameter(Mandatory=$true)]
+        [object[]] $Toolset
+    )
+
+    $goToolset = $Toolset  | Where-Object { ($_.name -eq "go") -and ($_.default -ne "") } `
+                           | Select-Object default, arch -First 1
+
+    if ($goToolset.default -ne $null) {
+        $goPath = Join-Path $Env:AGENT_TOOLSDIRECTORY "/go/$($goToolset.default)/$($goToolset.arch)" -Resolve
+
+        Write-Host "Use Go $($goToolset.default) as a system Go"
+        Add-MachinePathItem -PathItem "$goPath\bin" | Out-Null
+        # Set the GOROOT environment variable.
+        setx GOROOT "$goPath" /M | Out-Null
+    } else {
+        Write-Host "Default Go version not found in toolset file!"
+    }
+}
+
 $ErrorActionPreference = "Stop"
 
 Import-Module -Name ImageHelpers -Force
 
 # Get toolcache content from toolset
-$ToolsToInstall = @("Python", "Node", "Boost")
+$ToolsToInstall = @("Python", "Node", "Boost", "Go")
+
 $tools = Get-ToolsetContent | Select-Object -ExpandProperty toolcache | Where {$ToolsToInstall -contains $_.Name}
 
 foreach ($tool in $tools) {
@@ -80,4 +102,5 @@ foreach ($tool in $tools) {
 }
 
 # Install default python version
-Set-DefaultPythonVersion -Toolset $tools
+Set-DefaultPythonVersion -Toolset $tools
+Set-DefaultGoVersion -Toolset $tools

images/win/scripts/Installers/Validate-Go.ps1

@@ -1,50 +0,0 @@
-################################################################################
-##  File:  Validate-Go.ps1
-##  Desc:  Validate Go
-################################################################################
-
-# Function that gets the version of Go at the specified path
-function Get-GoVersion
-{
-    Param
-    (
-        [String]$goVersion
-    )
-    Write-Host "Check if $goVersion is presented in the system"
-    $destinationPath = "$($env:AGENT_TOOLSDIRECTORY)\go"
-    $goDirectory = Get-ChildItem -Path $destinationPath -Filter "$goVersion*" | Select-Object -First 1
-    $goPath = Join-Path $destinationPath "$goDirectory\x64"
-
-    $env:Path = "$goPath\bin;" + $env:Path
-    $version = $(go version)
-
-    $matchVersion = $version -match $goVersion
-    $semanticEquality = $version -match 'go version go(?<version>.*) win.*'
-
-    if($semanticEquality -And $matchVersion)
-    {
-        $goFullVersion = $Matches.version
-        Write-Host "$goFullVersion has been found"
-    }
-    else
-    {
-        Write-Host "Unable to determine Go version at " + $goPath
-        exit 1
-    }
-}
-
-# Verify that go.exe is on the path
-if(Get-Command -Name 'go')
-{
-    Write-Host "$(go version) is on the path."
-}
-else
-{
-    Write-Host "Go is not on the path."
-    exit 1
-}
-
-$goVersionsToInstall = $env:GO_VERSIONS.split(", ", [System.StringSplitOptions]::RemoveEmptyEntries)
-foreach($go in $goVersionsToInstall) {
-    Get-GoVersion -goVersion $go
-}

images/win/scripts/Installers/Validate-Toolset.ps1

@@ -9,13 +9,13 @@ function Run-ExecutableTests {
         [Parameter(Mandatory)] [string[]] $Executables,
         [Parameter(Mandatory)] [string] $ToolPath
     )
-
+    $versionCommand = $Executables["command"]
-    foreach ($executable in $Executables) {
+    foreach ($executable in $Executables["tools"]) {
         $executablePath = Join-Path $ToolPath $executable
 
         Write-Host "Check $executable..."
         if (Test-Path $executablePath) {
-            Write-Host "$executable is successfully installed: $(& $executablePath --version)"
+            Write-Host "$executable is successfully installed: $(& $executablePath $versionCommand)"
         } else {
             Write-Host "$executablePath is not installed!"
             exit 1
@@ -29,11 +29,12 @@ function Validate-SystemDefaultTool {
         [Parameter(Mandatory)] [string] $ExpectedVersion
     )
 
+    $versionCommand = $toolsExecutables[$ToolName]["command"]
     $binName = $ToolName.ToLower()
 
     # Check if tool on path
     if (Get-Command -Name $binName) {
-        $versionOnPath = $(& $binName --version 2>&1) | Select-String -Pattern ".*(\d+\.\d+\.\d+)"
+        $versionOnPath = $(& $binName $versionCommand 2>&1) | Select-String -Pattern ".*(\d+\.\d+[\.\d+]+)"
 
         # Check if version is correct
         if ($versionOnPath.matches.Groups[1].Value -notlike $ExpectedVersion) {
@@ -52,9 +53,22 @@ $ErrorActionPreference = "Stop"
 
 # Define executables for cached tools
 $toolsExecutables = @{
-    Python = @("python.exe", "Scripts\pip.exe")
+    Python = @{
-    node = @("node.exe", "npm")
+        tools = @("python.exe", "Scripts\pip.exe")
-    PyPy = @("python.exe", "Scripts\pip.exe")
+        command = "--version"
+    }
+    node = @{
+        tools = @("node.exe", "npm")
+        command = "--version"
+    }
+    PyPy = @{
+        tools = @("python.exe", "Scripts\pip.exe")
+        command = "--version"
+    }
+    go = @{
+        tools = @("bin\go.exe")
+        command = "version"
+    }
 }
 
 # Get toolcache content from toolset

images/win/toolset-2016.json

@@ -48,6 +48,21 @@
                 "14.*"
             ]
         },
+        {
+            "name": "go",
+            "url" : "https://raw.githubusercontent.com/actions/go-versions/master/versions-manifest.json",
+            "arch": "x64",
+            "platform" : "win32",
+            "versions": [
+                "1.9.*",
+                "1.10.*",
+                "1.11.*",
+                "1.12.*",
+                "1.13.*",
+                "1.14.*"
+            ],
+            "default": "1.14.*"
+        },
         {
             "name": "Boost",
             "url" : "https://raw.githubusercontent.com/actions/boost-versions/master/versions-manifest.json",

images/win/toolset-2019.json

@@ -48,9 +48,24 @@
                 "14.*"
             ]
         },
+        {
+            "name": "go",
+            "url" : "https://raw.githubusercontent.com/actions/go-versions/master/versions-manifest.json",
+            "arch": "x64",
+            "platform" : "win32",
+            "versions": [
+                "1.9.*",
+                "1.10.*",
+                "1.11.*",
+                "1.12.*",
+                "1.13.*",
+                "1.14.*"
+            ],
+            "default": "1.14.*"
+        },
         {
             "name": "Boost",
-            "url" : "https://raw.githubusercontent.com/akv-platform/boost-hostedtoolcache/master/versions-manifest.json",
+            "url" : "https://raw.githubusercontent.com/actions/boost-versions/master/versions-manifest.json",
             "arch": "x86_64",
             "platform" : "win32",
             "toolset": "msvc14.1",