Add Python tools installation from Github releases for Windows (#705)

* Add Toolset provision for Windows
Co-authored-by: MaksimZhukov <v-mazhuk@microsoft.com>

images/win/Windows2016-Azure.json

@@ -18,6 +18,7 @@
         "run_scan_antivirus": "false",
 
         "root_folder": "C:",
+        "toolset_json_path": "{{env `TEMP`}}\\toolset.json",
         "image_folder": "C:\\image",
         "commit_file": "C:\\image\\commit.txt",
         "imagedata_file": "C:\\imagedata.json",
@@ -318,6 +319,11 @@
             "source": "{{template_dir}}/toolcache-2016.json",
             "destination": "{{user `root_folder`}}/toolcache.json"
         },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolset-2016.json",
+            "destination": "{{user `toolset_json_path`}}"
+        },
         {
             "type": "powershell",
             "environment_vars":[
@@ -328,6 +334,15 @@
                 "{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1"
             ]
         },
+        {
+            "type": "powershell",
+            "environment_vars":[
+                "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
+            ],
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1"
+            ]
+        },
         {
             "type": "powershell",
             "scripts":[
@@ -642,6 +657,15 @@
                 "{{ template_dir }}/scripts/Installers/Validate-ToolCache.ps1"
             ]
         },
+        {
+            "type": "powershell",
+            "environment_vars":[
+                "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
+            ],
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Validate-Toolset.ps1"
+            ]
+        },
         {
             "type": "powershell",
             "scripts":[

images/win/Windows2019-Azure.json

@@ -18,6 +18,7 @@
         "run_scan_antivirus": "false",
 
         "root_folder": "C:",
+        "toolset_json_path": "{{env `TEMP`}}\\toolset.json",
         "image_folder": "C:\\image",
         "commit_file": "C:\\image\\commit.txt",
         "imagedata_file": "C:\\imagedata.json",
@@ -291,6 +292,11 @@
             "source": "{{template_dir}}/toolcache-2019.json",
             "destination": "{{user `root_folder`}}/toolcache.json"
         },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolset-2019.json",
+            "destination": "{{user `toolset_json_path`}}"
+        },
         {
             "type": "powershell",
             "environment_vars":[
@@ -301,6 +307,15 @@
                 "{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1"
             ]
         },
+        {
+            "type": "powershell",
+            "environment_vars":[
+                "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
+            ],
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1"
+            ]
+        },
         {
             "type": "powershell",
             "scripts":[
@@ -639,6 +654,15 @@
                 "{{ template_dir }}/scripts/Installers/Validate-ToolCache.ps1"
             ]
         },
+        {
+            "type": "powershell",
+            "environment_vars":[
+                "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
+            ],
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Validate-Toolset.ps1"
+            ]
+        },
         {
             "type": "powershell",
             "scripts":[

images/win/scripts/ImageHelpers/ImageHelpers.psm1

@@ -15,6 +15,7 @@ Export-ModuleMember -Function @(
     'Set-SystemVariable'
     'Install-Binary'
     'Get-ToolcachePackages'
+    'Get-ToolsetContent'
     'Get-ToolsByName'
     'Add-ContentToMarkdown'
     'Add-SoftwareDetailsToMarkdown'

images/win/scripts/ImageHelpers/InstallHelpers.ps1

@@ -283,6 +283,11 @@ function Get-ToolcachePackages {
     Get-Content -Raw $toolcachePath | ConvertFrom-Json
 }
 
+function Get-ToolsetContent {
+    $toolsetJson = Get-Content -Path $env:TOOLSET_JSON_PATH -Raw
+    ConvertFrom-Json -InputObject $toolsetJson
+}
+
 function Get-ToolsByName {
     param (
         [Parameter(Mandatory = $True)]

images/win/scripts/Installers/Download-ToolCache.ps1

@@ -44,21 +44,6 @@ Function NPMFeed-AuthSetup {
     $npmrcContent | Out-File -FilePath "$($env:TEMP)/.npmrc" -Encoding utf8
 }
 
-Function Set-DefaultPythonVersion {
-    param(
-        [Parameter(Mandatory=$true)]
-        [System.Version] $Version,
-        [System.String] $Arch = "x64"
-    )
-
-    $pythonPath = $Env:AGENT_TOOLSDIRECTORY + "/Python/${Version}*/${Arch}"
-    $pythonDir = Get-Item -Path $pythonPath
-
-    Write-Host "Use Python ${Version} as a system Python"
-    Add-MachinePathItem -PathItem $pythonDir.FullName
-    Add-MachinePathItem -PathItem "$($pythonDir.FullName)\Scripts"
-}
-
 Function Set-DefaultRubyVersion {
     param(
         [Parameter(Mandatory=$true)]
@@ -106,5 +91,4 @@ $ToolVersions.PSObject.Properties | ForEach-Object {
     }
 }
 
-Set-DefaultPythonVersion -Version "3.7"
 Set-DefaultRubyVersion -Version "2.5"

images/win/scripts/Installers/Install-Toolset.ps1

@@ -0,0 +1,76 @@
+################################################################################
+##  File:  Install-Toolset.ps1
+##  Team:  CI-Build
+##  Desc:  Install toolset
+################################################################################
+
+Function Install-Asset {
+    param(
+        [Parameter(Mandatory=$true)]
+        [object] $ReleaseAsset
+    )
+
+    $releaseAssetName = [System.IO.Path]::GetFileNameWithoutExtension($ReleaseAsset.filename)
+    $assetFolderPath = Join-Path $env:TEMP $releaseAssetName
+    $assetArchivePath = Start-DownloadWithRetry -Url $ReleaseAsset.download_url -Name $ReleaseAsset.filename
+
+    Write-Host "Extract $($ReleaseAsset.filename) content..."
+    7z.exe x $assetArchivePath -o"$assetFolderPath" -y | Out-Null
+
+    Write-Host "Invoke installation script..."
+    Push-Location -Path $assetFolderPath
+    Invoke-Expression .\setup.ps1
+    Pop-Location
+}
+
+Function Set-DefaultPythonVersion {
+    param(
+        [Parameter(Mandatory=$true)]
+        [object[]] $Toolset
+    )
+
+    $python = $Toolset  | Where-Object { ($_.name -eq "Python") -and ($_.default -ne "") } `
+                        | Select-Object default, arch -First 1
+
+    if ($python.default -ne $null) {
+        $pythonPath = Join-Path $Env:AGENT_TOOLSDIRECTORY "/Python/$($python.default)/$($python.arch)" -Resolve
+
+        Write-Host "Use Python $($python.default) as a system Python"
+        Add-MachinePathItem -PathItem $pythonPath
+        Add-MachinePathItem -PathItem "$pythonPath\Scripts"
+    } else {
+        Write-Host "Default Python version not found in toolset file!"
+    }
+}
+
+$ErrorActionPreference = "Stop"
+
+Import-Module -Name ImageHelpers -Force
+
+# Get toolcache content from toolset
+$tools = Get-ToolsetContent | Select-Object -ExpandProperty toolcache
+
+foreach ($tool in $tools) {
+    # Get versions manifest for current tool
+    $assets = Invoke-RestMethod $tool.url
+
+    # Get github release asset for each version
+    foreach ($toolVersion in $tool.versions) {
+        $asset = $assets | Where-Object version -like $toolVersion `
+                         | Sort-Object -Property {[version]$_.version} -Descending `
+                         | Select-Object -ExpandProperty files `
+                         | Where-Object { ($_.platform -eq $tool.platform) -and ($_.arch -eq $tool.arch) } `
+                         | Select-Object -First 1
+
+        Write-Host "Installing $($tool.name) $toolVersion $($tool.arch)..."
+        if ($asset -ne $null) {
+            Install-Asset -ReleaseAsset $asset
+        } else {
+            Write-Host "Asset was not found in versions manifest"
+            exit 1
+        }
+    }
+}
+
+# Install default python version
+Set-DefaultPythonVersion -Toolset $tools

images/win/scripts/Installers/Validate-ToolCache.ps1

@@ -63,32 +63,6 @@ function RunTestsByPath {
     }
 }
 
-function Get-SystemDefaultPython {
-    Write-Host "Validate system Python..."
-
-    if (Get-Command -Name 'python')
-    {
-        Write-Host "Python $(& python -V 2>&1) on path"
-    }
-    else
-    {
-        Write-Host "Python is not on path"
-        exit 1
-    }
-
-    $pythonBinVersion = $(& python -V 2>&1)
-    if ($pythonBinVersion -notlike "Python 3.*")
-    {
-        Write-Error "Python 3 is not in the PATH"
-        exit 1
-    }
-
-    $pythonBinOnPath = Split-Path -Path (Get-Command -Name 'python').Path
-    $description = GetDefaultToolDescription -SoftwareVersion $pythonBinVersion -SoftwareLocation $pythonBinOnPath
-
-    return $description
-}
-
 function Get-SystemDefaultRuby {
     Write-Host "Validate system Ruby..."
 
@@ -196,9 +170,6 @@ function ToolcacheTest {
         }
     }
 
-    if ($SoftwareName -contains "Python") {
-        $markdownDescription += Get-SystemDefaultPython
-    }
     if ($SoftwareName -contains "Ruby") {
         $markdownDescription += Get-SystemDefaultRuby
     }
@@ -206,10 +177,6 @@ function ToolcacheTest {
     Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $markdownDescription
 }
 
-# Python test
-$PythonTests = @("python.exe", "Scripts\pip.exe")
-ToolcacheTest -SoftwareName "Python" -ExecTests $PythonTests
-
 # PyPy test
 $PyPyTests = @("python.exe", "bin\pip.exe")
 ToolcacheTest -SoftwareName "PyPy" -ExecTests $PyPyTests

images/win/scripts/Installers/Validate-Toolset.ps1

@@ -0,0 +1,111 @@
+################################################################################
+##  File:  Validate-Toolset.ps1
+##  Team:  CI-Build
+##  Desc:  Validate Toolset
+################################################################################
+
+function Run-ExecutableTests {
+    param (
+        [Parameter(Mandatory)] [string[]] $Executables,
+        [Parameter(Mandatory)] [string] $ToolPath
+    )
+
+    foreach ($executable in $Executables) {
+        $executablePath = Join-Path $ToolPath $executable
+
+        Write-Host "Check $executable..."
+        if (Test-Path $executablePath) {
+            Write-Host "$executable is successfully installed: $(& $executablePath --version)"
+        } else {
+            Write-Host "$executablePath is not installed!"
+            exit 1
+        }
+    }
+}
+
+function Validate-SystemDefaultTool {
+    param (
+        [Parameter(Mandatory)] [string] $ToolName,
+        [Parameter(Mandatory)] [string] $ExpectedVersion
+    )
+
+    $binName = $ToolName.ToLower()
+
+    # Check if tool on path
+    if (Get-Command -Name $binName) {
+        $versionOnPath = $(& $binName --version 2>&1) | Select-String -Pattern ".*(\d+\.\d+\.\d+)"
+        $versionBinPath = Split-Path -Path (Get-Command -Name $binName).Path
+
+        # Check if version is correct
+        if ($versionOnPath.matches.Groups[1].Value -notlike $ExpectedVersion) {
+            Write-Error "$ToolName $ExpectedVersion is not in the PATH"
+            exit 1
+        }
+
+        Write-Host "$ToolName $versionOnPath on path"
+    } else {
+        Write-Host "$ToolName is not on path"
+        exit 1
+    }
+
+    # Add default version description to markdown
+    $description = "<br/>__System default version:__ $versionOnPath<br/>"
+    $description += "_Environment:_<br/>"
+    $description += "* Location: $versionBinPath<br/>"
+    $description += "* PATH: contains the location of $versionOnPath<br/>"
+
+    return $description
+}
+
+$ErrorActionPreference = "Stop"
+
+Import-Module -Name ImageHelpers -Force
+
+# Define executables for cached tools
+$toolsExecutables = @{ Python = @("python.exe", "Scripts\pip.exe") }
+
+# Get toolcache content from toolset
+$tools = Get-ToolsetContent | Select-Object -ExpandProperty toolcache
+
+foreach($tool in $tools) {
+    $markdownDescription = ""
+
+    $toolPath = Join-Path $env:AGENT_TOOLSDIRECTORY $tool.name
+    # Get executables for current tool
+    $toolExecs = $toolsExecutables[$tool.name]
+
+    foreach ($version in $tool.versions) {
+        # Check if version folder exists
+        $expectedVersionPath = Join-Path $toolPath $version
+        if (-not (Test-Path $expectedVersionPath)) {
+            Write-Host "Expected $($tool.name) $version folder is not found!"
+            exit 1
+        }
+
+        # Take latest installed version in case if toolset version contains wildcards
+        $foundVersion = Get-Item $expectedVersionPath `
+                        | Sort-Object -Property {[version]$_.name} -Descending `
+                        | Select-Object -First 1
+
+        # Check for required architecture folder
+        $foundVersionArchPath = Join-Path $foundVersion $tool.arch
+        if (-not (Test-Path $foundVersionArchPath)) {
+            Write-Host "Expected $($tool.name)($($tool.arch)) $($foundVersion.name) architecture folder is not found!"
+            exit 1
+        }
+
+        Write-Host "Run validation test for $($tool.name)($($tool.arch)) $($foundVersion.name) executables..."
+        Run-ExecutableTests -Executables $toolExecs -ToolPath $foundVersionArchPath
+
+        # Add to tool version to markdown
+        $markdownDescription += "_Version:_ $($foundVersion.name)<br/>"
+    }
+
+    # Create markdown description for system default tool
+    if (-not ([string]::IsNullOrEmpty($tool.default))) {
+        Write-Host "Validate system default $($tool.name)($($tool.arch)) $($tool.default)..."
+        $markdownDescription += Validate-SystemDefaultTool -ToolName $tool.name -ExpectedVersion $tool.default
+    }
+
+    Add-SoftwareDetailsToMarkdown -SoftwareName "$($tool.name) ($($tool.arch))" -DescriptionMarkdown $markdownDescription
+}

images/win/toolcache-2016.json

@@ -1,10 +1,4 @@
 {
-    "@actions/toolcache-python-windows-x64": [
-        "2.7", "3.5", "3.6", "3.7", "3.8"
-    ],
-    "@actions/toolcache-python-windows-x86": [
-        "2.7", "3.5", "3.6", "3.7", "3.8"
-    ],
     "@actions/toolcache-ruby-windows-x64": [
         "2.4", "2.5", "2.6", "2.7"
     ],

images/win/toolcache-2019.json

@@ -1,10 +1,4 @@
 {
-    "@actions/toolcache-python-windows-x64": [
-        "2.7", "3.5", "3.6", "3.7", "3.8"
-    ],
-    "@actions/toolcache-python-windows-x86": [
-        "2.7", "3.5", "3.6", "3.7", "3.8"
-    ],
     "@actions/toolcache-ruby-windows-x64": [
         "2.4", "2.5", "2.6", "2.7"
     ],

images/win/toolset-2016.json

@@ -0,0 +1,31 @@
+{
+    "toolcache": [
+        {
+            "name": "Python",
+            "url" : "https://raw.githubusercontent.com/actions/python-versions/master/versions-manifest.json",
+            "arch": "x64",
+            "platform" : "win32",
+            "versions": [
+                "2.7.*",
+                "3.5.*",
+                "3.6.*",
+                "3.7.*",
+                "3.8.*"
+            ],
+            "default": "3.7.*"
+        },
+        {
+            "name": "Python",
+            "url" : "https://raw.githubusercontent.com/actions/python-versions/master/versions-manifest.json",
+            "arch": "x86",
+            "platform" : "win32",
+            "versions": [
+                "2.7.*",
+                "3.5.*",
+                "3.6.*",
+                "3.7.*",
+                "3.8.*"
+            ]
+        }
+    ]
+}

images/win/toolset-2019.json

@@ -0,0 +1,31 @@
+{
+    "toolcache": [
+        {
+            "name": "Python",
+            "url" : "https://raw.githubusercontent.com/actions/python-versions/master/versions-manifest.json",
+            "arch": "x64",
+            "platform" : "win32",
+            "versions": [
+                "2.7.*",
+                "3.5.*",
+                "3.6.*",
+                "3.7.*",
+                "3.8.*"
+            ],
+            "default": "3.7.*"
+        },
+        {
+            "name": "Python",
+            "url" : "https://raw.githubusercontent.com/actions/python-versions/master/versions-manifest.json",
+            "arch": "x86",
+            "platform" : "win32",
+            "versions": [
+                "2.7.*",
+                "3.5.*",
+                "3.6.*",
+                "3.7.*",
+                "3.8.*"
+            ]
+        }
+    ]
+}