Switch provisioners to install Go from GitHub releases on Ubuntu (#1025)

* install go

* added validation

* check zero build version

* add versions and for ubuntu2004

* setup for ubuntu

* fix execute command

* fix issue with cat

* fix go.sh

* fix comments

* remove documentation

* fix comment position go.sh ubuntu2004

* fix version

* remove import

* remove variables

* remove fake url

* fix linking to default version

Co-authored-by: Dmitry Shibanov <v-dmshib@microsoft.com>

images/linux/scripts/installers/Install-Toolset.ps1

@@ -28,7 +28,8 @@ $ErrorActionPreference = "Stop"
 
 # Get toolset content
 $toolsetJson = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw
-$toolsToInstall = @("Python", "Node", "Boost")
+$toolsToInstall = @("Python", "Node", "Boost", "Go")
+
 $tools = ConvertFrom-Json -InputObject $toolsetJson | Select-Object -ExpandProperty toolcache | Where {$ToolsToInstall -contains $_.Name}
 
 foreach ($tool in $tools) {
@@ -54,4 +55,5 @@ foreach ($tool in $tools) {
 }
 
 chown -R "$($env:SUDO_USER):$($env:SUDO_USER)" /opt/hostedtoolcache/Python
-chown -R "$($env:SUDO_USER):$($env:SUDO_USER)" /opt/hostedtoolcache/node
+chown -R "$($env:SUDO_USER):$($env:SUDO_USER)" /opt/hostedtoolcache/node
+chown -R "$($env:SUDO_USER):$($env:SUDO_USER)" /opt/hostedtoolcache/go

images/linux/scripts/installers/Validate-Toolset.ps1

@@ -9,13 +9,13 @@ function Run-ExecutableTests {
         [Parameter(Mandatory)] [string[]] $Executables,
         [Parameter(Mandatory)] [string] $ToolPath
     )
+    $versionCommand = $Executables["command"]
 
-    foreach ($executable in $Executables) {
+    foreach ($executable in $Executables["tools"]) {
         $executablePath = Join-Path $ToolPath $executable
-
         Write-Host "Check $executable..."
         if (Test-Path $executablePath) {
-            Write-Host "$executable is successfully installed: $(& $executablePath --version)"
+            Write-Host "$executable is successfully installed: $(& $executablePath $versionCommand)"
         } else {
             Write-Host "$executablePath is not installed!"
             exit 1
@@ -27,9 +27,22 @@ $ErrorActionPreference = "Stop"
 
 # Define executables for cached tools
 $toolsExecutables = @{
-    Python = @("python", "bin/pip")
+    Python = @{
-    node = @("bin/node", "bin/npm")
+        tools = @("python", "bin/pip")
-    PyPy = @("bin/python", "bin/pip")
+        command = "--version"
+    }
+    node = @{
+        tools = @("bin/node", "bin/npm")
+        command = "--version"
+    }
+    PyPy = @{
+        tools = @("bin/python", "bin/pip")
+        command = "--version"
+    }
+    go = @{
+        tools = @("bin/go")
+        command = "version"
+    }
 }
 
 # Get toolset content

images/linux/scripts/installers/go.sh

@@ -4,55 +4,23 @@
 ##  Desc:  Installs go, configures GOROOT, and adds go to the path
 ################################################################################
 
-# Source the helpers for use with the script
+# Fail out if any setups fail
-source $HELPER_SCRIPTS/document.sh
+set -e
-golangTags="/tmp/golang_tags.json"
 
-# This function installs Go using the specified arguments:
+toolsetJson="$INSTALLER_SCRIPT_FOLDER/toolset.json"
-#   $1=MajorVersion (1.11)
+toolsetVersions=(`ls $AGENT_TOOLSDIRECTORY/go`)
-#   $2=IsDefaultVersion (true or false)
+defaultVersion=$(jq -r '.toolcache[] | select(.name | contains("go")) | .default' $toolsetJson)
-function InstallGo () {
-    version=$( getFullGoVersion $1 )
-    downloadVersion="go$version.linux-amd64.tar.gz"
-    goFolder="$AGENT_TOOLSDIRECTORY/go/$version/x64"
 
-    echo "Install Go $version"
+for toolsetVersion in ${toolsetVersions[@]}
-    curl -sL https://dl.google.com/go/${downloadVersion} -o ${downloadVersion}
+do
-    mkdir -p $goFolder
+    major="$(cut -d'.' -f1 <<< "$toolsetVersion")"
-    tar -C $goFolder -xzf $downloadVersion --strip-components=1 go
+    minor="$(cut -d'.' -f2 <<< "$toolsetVersion")"
-    rm $downloadVersion
+    goFolder="$AGENT_TOOLSDIRECTORY/go/$toolsetVersion/x64"
-    echo "GOROOT_${1//./_}_X64=$goFolder" | tee -a /etc/environment
-    DocumentInstalledItem "Go $version ($($goFolder/bin/go version))"
 
-    # Create symlink in old location /usr/local/go<version> to new location
+    echo "GOROOT_${major}_${minor}_X64=$goFolder" | tee -a /etc/environment
-    ln -s $goFolder /usr/local/go$version
 
-    # If this version of Go is to be the default version,
+    if [[ "$toolsetVersion" =~ $defaultVersion ]]; then
-    # symlink it into the path and point GOROOT to it.
-    if [ $2 = true ]
-    then
         ln -s $goFolder/bin/* /usr/bin/
         echo "GOROOT=$goFolder" | tee -a /etc/environment
     fi
-}
+done
-
-function getFullGoVersion () {
-    local pattern="refs/tags/go$1([.0-9]{0,3})$"
-    local query='[.[] | select( .ref | test($pattern))] | .[-1] | .ref'
-    refValue=$(jq --arg pattern "$pattern" "$query" $golangTags)
-    version=$(echo "$refValue" | cut -d '/' -f 3 | awk -F 'go' '{print $2}')
-    version=$(echo "${version//\"}") # 1.12.17
-    echo $version
-}
-
-# load golang_tags.json file
-curl -s 'https://api.github.com/repos/golang/go/git/refs/tags' > $golangTags
-# Install Go versions
-for go in ${GO_VERSIONS}; do
-    echo "Installing Go ${go}"
-    if [[ $go == $GO_DEFAULT ]]; then
-        InstallGo $go true
-    else
-        InstallGo $go false
-    fi
-done

images/linux/toolset-1604.json

@@ -35,6 +35,19 @@
                 "14.*"
             ]
         },
+        {
+            "name": "go",
+            "url" : "https://raw.githubusercontent.com/actions/go-versions/master/versions-manifest.json",
+            "arch": "x64",
+            "platform" : "linux",
+            "versions": [
+                "1.11.*",
+                "1.12.*",
+                "1.13.*",
+                "1.14.*"
+            ],
+            "default": "1.14.*"
+        },
         {
             "name": "boost",
             "url" : "https://raw.githubusercontent.com/actions/boost-versions/master/versions-manifest.json",

images/linux/toolset-1804.json

@@ -35,6 +35,19 @@
                 "14.*"
             ]
         },
+        {
+            "name": "go",
+            "url" : "https://raw.githubusercontent.com/actions/go-versions/master/versions-manifest.json",
+            "arch": "x64",
+            "platform" : "linux",
+            "versions": [
+                "1.11.*",
+                "1.12.*",
+                "1.13.*",
+                "1.14.*"
+            ],
+            "default": "1.14.*"
+        },
         {
             "name": "boost",
             "url" : "https://raw.githubusercontent.com/actions/boost-versions/master/versions-manifest.json",

images/linux/toolset-2004.json

@@ -34,6 +34,16 @@
                 "12.*",
                 "14.*"
             ]
+        },
+        {
+            "name": "go",
+            "url" : "https://raw.githubusercontent.com/actions/go-versions/master/versions-manifest.json",
+            "arch": "x64",
+            "platform" : "linux",
+            "versions": [
+                "1.14.*"
+            ],
+            "default": "1.14.*"
         }
     ]
 }

images/linux/ubuntu1604.json

@@ -24,9 +24,7 @@
         "image_version": "dev",
         "image_os": "ubuntu16",
         "github_feed_token": null,
-        "run_validation_diskspace": "false",
+        "run_validation_diskspace": "false"
-        "go_default": "1.14",
-        "go_versions": "1.11 1.12 1.13 1.14"
     },
     "sensitive-variables": ["client_secret", "github_feed_token"],
     "builders": [
@@ -276,19 +274,6 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
-        {
-            "type": "shell",
-            "scripts": [
-                "{{template_dir}}/scripts/installers/go.sh"
-            ],
-            "environment_vars": [
-                "METADATA_FILE={{user `metadata_file`}}",
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
-                "GO_VERSIONS={{user `go_versions`}}",
-                "GO_DEFAULT={{user `go_default`}}"
-            ],
-            "execute_command": "chmod +x {{ .Path }}; sudo {{ .Vars }} {{ .Path }}"
-        },
         {
             "type": "shell",
             "scripts":[
@@ -302,6 +287,16 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} pwsh -f {{ .Path }}'"
         },
+        {
+            "type": "shell",
+            "scripts": [
+                "{{template_dir}}/scripts/installers/go.sh"
+            ],
+            "environment_vars": [
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}"
+            ],
+            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+        },
         {
             "type": "shell",
             "scripts":[

images/linux/ubuntu1804.json

@@ -24,9 +24,7 @@
         "image_version": "dev",
         "image_os": "ubuntu18",
         "github_feed_token": null,
-        "run_validation_diskspace": "false",
+        "run_validation_diskspace": "false"
-        "go_default": "1.14",
-        "go_versions": "1.11 1.12 1.13 1.14"
     },
     "sensitive-variables": ["client_secret", "github_feed_token"],
     "builders": [
@@ -280,19 +278,6 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
-        {
-            "type": "shell",
-            "scripts": [
-                "{{template_dir}}/scripts/installers/go.sh"
-            ],
-            "environment_vars": [
-                "METADATA_FILE={{user `metadata_file`}}",
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
-                "GO_VERSIONS={{user `go_versions`}}",
-                "GO_DEFAULT={{user `go_default`}}"
-            ],
-            "execute_command": "chmod +x {{ .Path }}; sudo {{ .Vars }} {{ .Path }}"
-        },
         {
             "type": "shell",
             "scripts":[
@@ -306,6 +291,16 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} pwsh -f {{ .Path }}'"
         },
+        {
+            "type": "shell",
+            "scripts": [
+                "{{template_dir}}/scripts/installers/go.sh"
+            ],
+            "environment_vars": [
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}"
+            ],
+            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+        },
         {
             "type": "shell",
             "scripts":[

images/linux/ubuntu2004.json

@@ -279,19 +279,6 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
-        {
-            "type": "shell",
-            "scripts": [
-                "{{template_dir}}/scripts/installers/go.sh"
-            ],
-            "environment_vars": [
-                "METADATA_FILE={{user `metadata_file`}}",
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
-                "GO_VERSIONS={{user `go_versions`}}",
-                "GO_DEFAULT={{user `go_default`}}"
-            ],
-            "execute_command": "chmod +x {{ .Path }}; sudo {{ .Vars }} {{ .Path }}"
-        },
         {
             "type": "shell",
             "scripts":[
@@ -305,6 +292,16 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} pwsh -f {{ .Path }}'"
         },
+        {
+            "type": "shell",
+            "scripts": [
+                "{{template_dir}}/scripts/installers/go.sh"
+            ],
+            "environment_vars": [
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}"
+            ],
+            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+        },
         {
             "type": "shell",
             "scripts":[