Install PyPy from official site instead of NPM for Windows (#873)

* install PyPy from official repo

images/win/Windows2016-Azure.json

@@ -334,6 +334,15 @@
                 "{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1"
             ]
         },
+        {
+            "type": "powershell",
+            "environment_vars":[
+                "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
+            ],
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Install-PyPy.ps1"
+            ]
+        },
         {
             "type": "powershell",
             "environment_vars":[

images/win/Windows2019-Azure.json

@@ -307,6 +307,15 @@
                 "{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1"
             ]
         },
+        {
+            "type": "powershell",
+            "environment_vars":[
+                "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
+            ],
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Install-PyPy.ps1"
+            ]
+        },
         {
             "type": "powershell",
             "environment_vars":[

images/win/scripts/Installers/Install-PyPy.ps1

@@ -0,0 +1,120 @@
+################################################################################
+##  File:  Install-PyPy.ps1
+##  Team:  CI-Build
+##  Desc:  Install PyPy
+################################################################################
+
+function Get-PyPyVersions
+{
+    $uri = "https://api.bitbucket.org/2.0/repositories/pypy/pypy/downloads?pagelen=100"
+    try
+    {
+        (Invoke-RestMethod -Uri $uri).Values
+    }
+    catch
+    {
+        Write-Host "Enable to send request to the '$uri'. Error: '$_'"
+        exit 1
+    }
+}
+
+function Install-PyPy
+{
+    param(
+        [String]$PackagePath,
+        [String]$Architecture
+    )
+
+    # Expand archive with binaries
+    $packageName = [IO.Path]::GetFileNameWithoutExtension((Split-Path -Path $packagePath -Leaf))
+    $tempFolder = Join-Path -Path $env:Temp -ChildPath $packageName
+    Extract-7Zip -Path $packagePath -DestinationPath $env:Temp
+
+    # Get Python version from binaries
+    $pypyApp = Get-ChildItem -Path "$tempFolder\pypy*.exe" | Where-Object Name -match "pypy(\d+)?.exe"
+    $pypyName = $pypyApp.Name
+    $pypyVersion = & $pypyApp -c "import sys;print('{}.{}.{}'.format(sys.version_info[0],sys.version_info[1],sys.version_info[2]))"
+
+    if ($pypyVersion)
+    {
+        Write-Host "Installing PyPy $pypyVersion"
+        $pypyToolcachePath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "PyPy"
+        $pypyVersionPath = Join-Path -Path $pypyToolcachePath -ChildPath $pypyVersion
+        $pypyArchPath = Join-Path -Path $pypyVersionPath -ChildPath $architecture
+
+        if (-not (Test-Path $pypyToolcachePath)) {
+            Write-Host "Create PyPy toolcache folder"
+            New-Item -ItemType Directory -Path $pypyToolcachePath | Out-Null
+        }
+
+        Write-Host "Create PyPy '${pypyVersion}' folder in '${pypyVersionPath}'"
+        New-Item -ItemType Directory -Path $pypyVersionPath -Force | Out-Null
+
+        Write-Host "Move PyPy '${pypyVersion}' files to '${pypyArchPath}'"
+        Move-Item -Path $tempFolder -Destination $pypyArchPath | Out-Null
+
+        Write-Host "Install PyPy '${pypyVersion}' in '${pypyArchPath}'"
+        cmd.exe /c "cd /d $pypyArchPath && mklink python.exe $pypyName && python.exe -m ensurepip && python.exe -m pip install --upgrade pip"
+
+        if ($LASTEXITCODE -ne 0)
+        {
+            Throw "Error happened during PyPy installation"
+            exit 1
+        }
+
+        # https://github.com/actions/setup-python/blob/master/src/find-python.ts
+        # https://github.com/microsoft/azure-pipelines-tasks/blob/master/Tasks/UsePythonVersionV0/usepythonversion.ts
+        #  // For PyPy, Windows uses 'bin', not 'Scripts'.
+        # const _binDir = path.join(installDir, 'bin');
+        # PyPy v7.3.1 or higher creates only Scripts folder therefore to preserve back compatibility with UsePythonVersionV0 task
+        # We should create a Scripts -> bin symlink
+        Write-Host "Symbolic link created for '$pypyArchPath\Scripts' <<===>> '$pypyArchPath\bin'"
+        New-Item -Path "$pypyArchPath\bin" -ItemType SymbolicLink -Value "$pypyArchPath\Scripts" | Out-Null
+
+        Write-Host "Create complete file"
+        New-Item -ItemType File -Path $pypyVersionPath -Name "$architecture.complete" | Out-Null
+    }
+    else
+    {
+        Write-Host "PyPy application is not found. Failed to expand '$packagePath' archive"
+        exit 1
+    }
+}
+
+$ErrorActionPreference = "Stop"
+Import-Module -Name ImageHelpers -Force -DisableNameChecking
+
+# Get PyPy content from toolset
+$pypyTools = Get-ToolsetContent | Select-Object -ExpandProperty toolcache | Where-Object Name -eq "PyPy"
+
+# Get PyPy versions from the repo
+$pypyVersions = Get-PyPyVersions
+
+Write-Host "Starting installation PyPy..."
+foreach($pypyTool in $pypyTools)
+{
+    foreach($pypyVersion in $pypyTool.versions)
+    {
+        # Query latest PyPy version
+        $filter = '{0}{1}-*-{2}.zip' -f $pypyTool.name, $pypyVersion, $pypyTool.platform
+        $latestMajorPyPyVersion = $pypyVersions | Where-Object {
+            $_.name -like $filter -and $_.name.Split('-')[1].Substring(1) -as [System.Version]
+        } | Sort-Object {[System.Version]$_.name.Split('-')[1].Substring(1)}  | Select-Object -Last 1
+
+        if ($latestMajorPyPyVersion)
+        {
+            $packageName = $latestMajorPyPyVersion.name
+            $packageDate = $latestMajorPyPyVersion.created_on
+
+            Write-Host "Found PyPy '$packageName' package created on '$packageDate'"
+            $url = $latestMajorPyPyVersion.links.self.href
+            $tempPyPyPackagePath = Start-DownloadWithRetry -Url $url -Name  $packageName
+            Install-PyPy -PackagePath $tempPyPyPackagePath -Architecture $pypyTool.arch
+        }
+        else
+        {
+            Write-Host "Failed to query PyPy version '$pypyVersion' by '$filter' filter"
+            exit 1
+        }
+    }
+}

images/win/scripts/Installers/Install-Toolset.ps1

@@ -48,7 +48,8 @@ $ErrorActionPreference = "Stop"
 Import-Module -Name ImageHelpers -Force
 
 # Get toolcache content from toolset
-$tools = Get-ToolsetContent | Select-Object -ExpandProperty toolcache
+$ToolsToInstall = @("Python", "Node")
+$tools = Get-ToolsetContent | Select-Object -ExpandProperty toolcache | Where {$ToolsToInstall -contains $_.Name}
 
 foreach ($tool in $tools) {
     # Get versions manifest for current tool

images/win/scripts/Installers/Validate-ToolCache.ps1

@@ -177,10 +177,6 @@ function ToolcacheTest {
     Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $markdownDescription
 }
 
-# PyPy test
-$PyPyTests = @("python.exe", "bin\pip.exe")
-ToolcacheTest -SoftwareName "PyPy" -ExecTests $PyPyTests
-
 # Ruby test
 $RubyTests = @("bin\ruby.exe")
 ToolcacheTest -SoftwareName "Ruby" -ExecTests $RubyTests

images/win/scripts/Installers/Validate-Toolset.ps1

@@ -63,8 +63,9 @@ Import-Module -Name ImageHelpers -Force
 
 # Define executables for cached tools
 $toolsExecutables = @{
-    Python = @("python.exe", "Scripts\pip.exe");
+    Python = @("python.exe", "Scripts\pip.exe")
     node = @("node.exe", "npm")
+    PyPy = @("python.exe", "Scripts\pip.exe")
 }
 
 # Get toolcache content from toolset
@@ -78,6 +79,11 @@ foreach($tool in $tools) {
     $toolExecs = $toolsExecutables[$tool.name]
 
     foreach ($version in $tool.versions) {
+        # Add wildcard if missing
+        if (-not $version.Contains('*')) {
+            $version += '.*'
+        }
+
         # Check if version folder exists
         $expectedVersionPath = Join-Path $toolPath $version
         if (-not (Test-Path $expectedVersionPath)) {
@@ -100,8 +106,14 @@ foreach($tool in $tools) {
         Write-Host "Run validation test for $($tool.name)($($tool.arch)) $($foundVersion.name) executables..."
         Run-ExecutableTests -Executables $toolExecs -ToolPath $foundVersionArchPath
 
+        $foundVersionName = $foundVersion.name
+        if ($tool.name -eq 'PyPy')
+        {
+            $pypyVersion = & "$foundVersionArchPath\python.exe" -c "import sys;print(sys.version.split('\n')[1])"
+            $foundVersionName = "{0} {1}" -f $foundVersionName, $pypyVersion
+        }
         # Add to tool version to markdown
-        $markdownDescription += "_Version:_ $($foundVersion.name)<br/>"
+        $markdownDescription += "_Version:_ $foundVersionName<br/>"
     }
 
     # Create markdown description for system default tool

images/win/toolcache-2016.json

@@ -2,9 +2,6 @@
     "@actions/toolcache-ruby-windows-x64": [
         "2.4", "2.5", "2.6", "2.7"
     ],
-    "@actions/toolcache-pypy-windows-x86": [
-        "2", "3"
-    ],
     "@actions/toolcache-boost-windows-msvc-14.1-x32-x64": [
         "1.69", "1.72"
     ]

images/win/toolcache-2019.json

@@ -2,9 +2,6 @@
     "@actions/toolcache-ruby-windows-x64": [
         "2.4", "2.5", "2.6", "2.7"
     ],
-    "@actions/toolcache-pypy-windows-x86": [
-        "2", "3"
-    ],
     "@actions/toolcache-boost-windows-msvc-14.1-x32-x64": [
         "1.69"
     ],

images/win/toolset-2016.json

@@ -27,6 +27,15 @@
                 "3.8.*"
             ]
         },
+        {
+            "name": "PyPy",
+            "arch": "x86",
+            "platform" : "win32",
+            "versions": [
+                "2.7",
+                "3.6"
+            ]
+        },
         {
             "name": "node",
             "url" : "https://raw.githubusercontent.com/actions/node-versions/master/versions-manifest.json",

images/win/toolset-2019.json

@@ -27,6 +27,15 @@
                 "3.8.*"
             ]
         },
+        {
+            "name": "PyPy",
+            "arch": "x86",
+            "platform" : "win32",
+            "versions": [
+                "2.7",
+                "3.6"
+            ]
+        },
         {
             "name": "node",
             "url" : "https://raw.githubusercontent.com/actions/node-versions/master/versions-manifest.json",