Add Python toolcache installation from Github releases for Ubuntu (#704)

Change logic to install Python to Ubuntu images to use GitHub releases from https://github.com/actions/python-versions

images/linux/scripts/installers/Install-Toolset.ps1

@@ -0,0 +1,55 @@
+################################################################################
+##  File:  Install-Toolset.ps1
+##  Team:  CI-Build
+##  Desc:  Install toolset
+################################################################################
+
+Function Install-Asset {
+    param(
+        [Parameter(Mandatory = $true)]
+        [object] $ReleaseAsset
+    )
+
+    Write-Host "Download $($ReleaseAsset.filename)"
+    wget $ReleaseAsset.download_url -nv --retry-connrefused --tries=10
+
+    Write-Host "Extract $($ReleaseAsset.filename) content..."
+    $assetFolderPath = Join-Path $env:INSTALLER_SCRIPT_FOLDER $($ReleaseAsset.filename)
+    New-Item -ItemType Directory -Path $assetFolderPath
+    tar -xzf $ReleaseAsset.filename -C $assetFolderPath
+
+    Write-Host "Invoke installation script..."
+    Push-Location -Path $assetFolderPath
+    Invoke-Expression "bash ./setup.sh"
+    Pop-Location
+}
+
+$ErrorActionPreference = "Stop"
+
+# Get toolset content
+$toolsetJson = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw
+$tools = ConvertFrom-Json -InputObject $toolsetJson | Select-Object -ExpandProperty toolcache
+
+foreach ($tool in $tools) {
+    # Get versions manifest for current tool
+    $assets = Invoke-RestMethod $tool.url
+
+    # Get github release asset for each version
+    foreach ($toolVersion in $tool.versions) {
+        $asset = $assets | Where-Object version -like $toolVersion `
+        | Select-Object -ExpandProperty files `
+        | Where-Object { ($_.platform -eq $tool.platform) -and ($_.platform_version -eq $tool.platform_version)} `
+        | Select-Object -First 1
+
+        Write-Host "Installing $($tool.name) $toolVersion $($tool.arch)..."
+        if ($asset -ne $null) {
+            Install-Asset -ReleaseAsset $asset
+        }
+        else {
+            Write-Host "Asset was not found in versions manifest"
+            exit 1
+        }
+    }
+}
+
+chown -R "$($env:SUDO_USER):$($env:SUDO_USER)" /opt/hostedtoolcache/Python

images/linux/scripts/installers/Validate-Toolset.ps1

@@ -0,0 +1,62 @@
+################################################################################
+##  File:  Validate-Toolset.ps1
+##  Team:  CI-Build
+##  Desc:  Validate Toolset
+################################################################################
+
+function Run-ExecutableTests {
+    param (
+        [Parameter(Mandatory)] [string[]] $Executables,
+        [Parameter(Mandatory)] [string] $ToolPath
+    )
+
+    foreach ($executable in $Executables) {
+        $executablePath = Join-Path $ToolPath $executable
+
+        Write-Host "Check $executable..."
+        if (Test-Path $executablePath) {
+            Write-Host "$executable is successfully installed: $(& $executablePath --version)"
+        } else {
+            Write-Host "$executablePath is not installed!"
+            exit 1
+        }
+    }
+}
+
+$ErrorActionPreference = "Stop"
+
+# Define executables for cached tools
+$toolsExecutables = @{ Python = @("python", "bin/pip") }
+
+# Get toolset content
+$toolsetJson = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw
+$tools = ConvertFrom-Json -InputObject $toolsetJson | Select-Object -ExpandProperty toolcache
+
+foreach($tool in $tools) {
+    Invoke-Expression "bash -c `"source $env:HELPER_SCRIPTS/document.sh; DocumentInstalledItem '$($tool.name):'`""
+
+    $toolPath = Join-Path $env:AGENT_TOOLSDIRECTORY $tool.name
+    # Get executables for current tool
+    $toolExecs = $toolsExecutables[$tool.name]
+
+    foreach ($version in $tool.versions) {
+        # Check if version folder exists
+        $expectedVersionPath = Join-Path $toolPath $version
+        if (-not (Test-Path $expectedVersionPath)) {
+            Write-Host "Expected $($tool.name) $version folder is not found!"
+            exit 1
+        }
+
+        # Take latest installed version in case if toolset version contains wildcards
+        $foundVersion = Get-Item $expectedVersionPath `
+                        | Sort-Object -Property {[version]$_.name} -Descending `
+                        | Select-Object -First 1
+        $foundVersionPath = Join-Path $foundVersion $tool.arch
+
+        Write-Host "Run validation test for $($tool.name)($($tool.arch)) $($foundVersion.name) executables..."
+        Run-ExecutableTests -Executables $toolExecs -ToolPath $foundVersionPath
+
+        # Add tool version to documentation
+        Invoke-Expression "bash -c `"source $env:HELPER_SCRIPTS/document.sh; DocumentInstalledItemIndent '$($tool.name) $($foundVersion.name)'`""
+    }
+}

images/linux/scripts/installers/hosted-tool-cache.sh

@@ -49,12 +49,6 @@ done;
 
 popd
 
-DocumentInstalledItem "Python:"
-pythons=$(ls $AGENT_TOOLSDIRECTORY/Python)
-for python in $pythons; do
-	DocumentInstalledItemIndent "Python $python"
-done;
-
 DocumentInstalledItem "Ruby:"
 rubys=$(ls $AGENT_TOOLSDIRECTORY/Ruby)
 for ruby in $rubys; do

images/linux/scripts/installers/test-toolcache.sh

@@ -69,6 +69,5 @@ done;
 
 AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache
 
-Test_Hostedtoolcache_Tool "Python" "x64/python -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+'"
 Test_Hostedtoolcache_Tool "Ruby" "x64/bin/ruby -e 'puts RUBY_VERSION' | egrep -o '[0-9]+\.[0-9]+'"
 Test_Hostedtoolcache_Tool "PyPy" "x64/bin/python -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+' | cut -d '.' -f 1"

images/linux/toolcache-1604.json

@@ -1,7 +1,4 @@
 {
-    "@actions/toolcache-python-ubuntu-1604-x64": [
-        "2.7", "3.5", "3.6", "3.7", "3.8"
-    ],
     "@actions/toolcache-ruby-ubuntu-1604-x64": [
         "2.4", "2.5", "2.6", "2.7"
     ],

images/linux/toolcache-1804.json

@@ -1,7 +1,4 @@
 {
-    "@actions/toolcache-python-ubuntu-1804-x64": [
-        "2.7", "3.5", "3.6", "3.7", "3.8"
-    ],
     "@actions/toolcache-ruby-ubuntu-1804-x64": [
         "2.4", "2.5", "2.6", "2.7"
     ],

images/linux/toolset-1604.json

@@ -0,0 +1,18 @@
+{
+    "toolcache": [
+        {
+            "name": "Python",
+            "url" : "https://raw.githubusercontent.com/actions/python-versions/master/versions-manifest.json",
+            "platform" : "linux",
+            "platform_version": "16.04",
+            "arch": "x64",
+            "versions": [
+                "2.7.*",
+                "3.5.*",
+                "3.6.*",
+                "3.7.*",
+                "3.8.*"
+            ]
+        }
+    ]
+}

images/linux/toolset-1804.json

@@ -0,0 +1,18 @@
+{
+    "toolcache": [
+        {
+            "name": "Python",
+            "url" : "https://raw.githubusercontent.com/actions/python-versions/master/versions-manifest.json",
+            "platform" : "linux",
+            "platform_version": "18.04",
+            "arch": "x64",
+            "versions": [
+                "2.7.*",
+                "3.5.*",
+                "3.6.*",
+                "3.7.*",
+                "3.8.*"
+            ]
+        }
+    ]
+}

images/linux/ubuntu1604.json

@@ -269,6 +269,11 @@
             "source": "{{template_dir}}/toolcache-1604.json",
             "destination": "{{user `installer_script_folder`}}/toolcache.json"
         },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolset-1604.json",
+            "destination": "{{user `installer_script_folder`}}/toolset.json"
+        },
         {
             "type": "shell",
             "scripts":[
@@ -287,6 +292,19 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
+        {
+            "type": "shell",
+            "scripts":[
+                "{{template_dir}}/scripts/installers/Install-Toolset.ps1",
+                "{{template_dir}}/scripts/installers/Validate-Toolset.ps1"
+            ],
+            "environment_vars": [
+                "METADATA_FILE={{user `metadata_file`}}",
+                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}"
+            ],
+            "execute_command": "sudo sh -c '{{ .Vars }} pwsh -f {{ .Path }}'"
+        },
         {
             "type": "shell",
             "scripts":[

images/linux/ubuntu1804.json

@@ -273,6 +273,11 @@
             "source": "{{template_dir}}/toolcache-1804.json",
             "destination": "{{user `installer_script_folder`}}/toolcache.json"
         },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolset-1804.json",
+            "destination": "{{user `installer_script_folder`}}/toolset.json"
+        },
         {
             "type": "shell",
             "scripts":[
@@ -291,6 +296,19 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
+        {
+            "type": "shell",
+            "scripts":[
+                "{{template_dir}}/scripts/installers/Install-Toolset.ps1",
+                "{{template_dir}}/scripts/installers/Validate-Toolset.ps1"
+            ],
+            "environment_vars": [
+                "METADATA_FILE={{user `metadata_file`}}",
+                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}"
+            ],
+            "execute_command": "sudo sh -c '{{ .Vars }} pwsh -f {{ .Path }}'"
+        },
         {
             "type": "shell",
             "scripts":[