From 3f4ad2f912e7627aee21fcf8ba29d351993e50cd Mon Sep 17 00:00:00 2001
From: Aleksandr Chebotov <47745270+al-cheb@users.noreply.github.com>
Date: Mon, 18 May 2020 11:08:06 +0300
Subject: [PATCH] Install PyPy from official site instead of NPM for Windows
(#873)
* install PyPy from official repo
---
images/win/Windows2016-Azure.json | 9 ++
images/win/Windows2019-Azure.json | 9 ++
.../win/scripts/Installers/Install-PyPy.ps1 | 120 ++++++++++++++++++
.../scripts/Installers/Install-Toolset.ps1 | 3 +-
.../scripts/Installers/Validate-ToolCache.ps1 | 4 -
.../scripts/Installers/Validate-Toolset.ps1 | 16 ++-
images/win/toolcache-2016.json | 3 -
images/win/toolcache-2019.json | 3 -
images/win/toolset-2016.json | 9 ++
images/win/toolset-2019.json | 9 ++
10 files changed, 172 insertions(+), 13 deletions(-)
create mode 100644 images/win/scripts/Installers/Install-PyPy.ps1
diff --git a/images/win/Windows2016-Azure.json b/images/win/Windows2016-Azure.json
index f05e79eb7..8c29d1559 100644
--- a/images/win/Windows2016-Azure.json
+++ b/images/win/Windows2016-Azure.json
@@ -334,6 +334,15 @@
"{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1"
]
},
+ {
+ "type": "powershell",
+ "environment_vars":[
+ "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
+ ],
+ "scripts":[
+ "{{ template_dir }}/scripts/Installers/Install-PyPy.ps1"
+ ]
+ },
{
"type": "powershell",
"environment_vars":[
diff --git a/images/win/Windows2019-Azure.json b/images/win/Windows2019-Azure.json
index 78d190f0b..deecbe6f6 100644
--- a/images/win/Windows2019-Azure.json
+++ b/images/win/Windows2019-Azure.json
@@ -307,6 +307,15 @@
"{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1"
]
},
+ {
+ "type": "powershell",
+ "environment_vars":[
+ "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
+ ],
+ "scripts":[
+ "{{ template_dir }}/scripts/Installers/Install-PyPy.ps1"
+ ]
+ },
{
"type": "powershell",
"environment_vars":[
diff --git a/images/win/scripts/Installers/Install-PyPy.ps1 b/images/win/scripts/Installers/Install-PyPy.ps1
new file mode 100644
index 000000000..10e55c2ed
--- /dev/null
+++ b/images/win/scripts/Installers/Install-PyPy.ps1
@@ -0,0 +1,120 @@
+################################################################################
+## File: Install-PyPy.ps1
+## Team: CI-Build
+## Desc: Install PyPy
+################################################################################
+
+function Get-PyPyVersions
+{
+ $uri = "https://api.bitbucket.org/2.0/repositories/pypy/pypy/downloads?pagelen=100"
+ try
+ {
+ (Invoke-RestMethod -Uri $uri).Values
+ }
+ catch
+ {
+ Write-Host "Enable to send request to the '$uri'. Error: '$_'"
+ exit 1
+ }
+}
+
+function Install-PyPy
+{
+ param(
+ [String]$PackagePath,
+ [String]$Architecture
+ )
+
+ # Expand archive with binaries
+ $packageName = [IO.Path]::GetFileNameWithoutExtension((Split-Path -Path $packagePath -Leaf))
+ $tempFolder = Join-Path -Path $env:Temp -ChildPath $packageName
+ Extract-7Zip -Path $packagePath -DestinationPath $env:Temp
+
+ # Get Python version from binaries
+ $pypyApp = Get-ChildItem -Path "$tempFolder\pypy*.exe" | Where-Object Name -match "pypy(\d+)?.exe"
+ $pypyName = $pypyApp.Name
+ $pypyVersion = & $pypyApp -c "import sys;print('{}.{}.{}'.format(sys.version_info[0],sys.version_info[1],sys.version_info[2]))"
+
+ if ($pypyVersion)
+ {
+ Write-Host "Installing PyPy $pypyVersion"
+ $pypyToolcachePath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "PyPy"
+ $pypyVersionPath = Join-Path -Path $pypyToolcachePath -ChildPath $pypyVersion
+ $pypyArchPath = Join-Path -Path $pypyVersionPath -ChildPath $architecture
+
+ if (-not (Test-Path $pypyToolcachePath)) {
+ Write-Host "Create PyPy toolcache folder"
+ New-Item -ItemType Directory -Path $pypyToolcachePath | Out-Null
+ }
+
+ Write-Host "Create PyPy '${pypyVersion}' folder in '${pypyVersionPath}'"
+ New-Item -ItemType Directory -Path $pypyVersionPath -Force | Out-Null
+
+ Write-Host "Move PyPy '${pypyVersion}' files to '${pypyArchPath}'"
+ Move-Item -Path $tempFolder -Destination $pypyArchPath | Out-Null
+
+ Write-Host "Install PyPy '${pypyVersion}' in '${pypyArchPath}'"
+ cmd.exe /c "cd /d $pypyArchPath && mklink python.exe $pypyName && python.exe -m ensurepip && python.exe -m pip install --upgrade pip"
+
+ if ($LASTEXITCODE -ne 0)
+ {
+ Throw "Error happened during PyPy installation"
+ exit 1
+ }
+
+ # https://github.com/actions/setup-python/blob/master/src/find-python.ts
+ # https://github.com/microsoft/azure-pipelines-tasks/blob/master/Tasks/UsePythonVersionV0/usepythonversion.ts
+ # // For PyPy, Windows uses 'bin', not 'Scripts'.
+ # const _binDir = path.join(installDir, 'bin');
+ # PyPy v7.3.1 or higher creates only Scripts folder therefore to preserve back compatibility with UsePythonVersionV0 task
+ # We should create a Scripts -> bin symlink
+ Write-Host "Symbolic link created for '$pypyArchPath\Scripts' <<===>> '$pypyArchPath\bin'"
+ New-Item -Path "$pypyArchPath\bin" -ItemType SymbolicLink -Value "$pypyArchPath\Scripts" | Out-Null
+
+ Write-Host "Create complete file"
+ New-Item -ItemType File -Path $pypyVersionPath -Name "$architecture.complete" | Out-Null
+ }
+ else
+ {
+ Write-Host "PyPy application is not found. Failed to expand '$packagePath' archive"
+ exit 1
+ }
+}
+
+$ErrorActionPreference = "Stop"
+Import-Module -Name ImageHelpers -Force -DisableNameChecking
+
+# Get PyPy content from toolset
+$pypyTools = Get-ToolsetContent | Select-Object -ExpandProperty toolcache | Where-Object Name -eq "PyPy"
+
+# Get PyPy versions from the repo
+$pypyVersions = Get-PyPyVersions
+
+Write-Host "Starting installation PyPy..."
+foreach($pypyTool in $pypyTools)
+{
+ foreach($pypyVersion in $pypyTool.versions)
+ {
+ # Query latest PyPy version
+ $filter = '{0}{1}-*-{2}.zip' -f $pypyTool.name, $pypyVersion, $pypyTool.platform
+ $latestMajorPyPyVersion = $pypyVersions | Where-Object {
+ $_.name -like $filter -and $_.name.Split('-')[1].Substring(1) -as [System.Version]
+ } | Sort-Object {[System.Version]$_.name.Split('-')[1].Substring(1)} | Select-Object -Last 1
+
+ if ($latestMajorPyPyVersion)
+ {
+ $packageName = $latestMajorPyPyVersion.name
+ $packageDate = $latestMajorPyPyVersion.created_on
+
+ Write-Host "Found PyPy '$packageName' package created on '$packageDate'"
+ $url = $latestMajorPyPyVersion.links.self.href
+ $tempPyPyPackagePath = Start-DownloadWithRetry -Url $url -Name $packageName
+ Install-PyPy -PackagePath $tempPyPyPackagePath -Architecture $pypyTool.arch
+ }
+ else
+ {
+ Write-Host "Failed to query PyPy version '$pypyVersion' by '$filter' filter"
+ exit 1
+ }
+ }
+}
diff --git a/images/win/scripts/Installers/Install-Toolset.ps1 b/images/win/scripts/Installers/Install-Toolset.ps1
index 25bfe58ec..bc2aa59d2 100644
--- a/images/win/scripts/Installers/Install-Toolset.ps1
+++ b/images/win/scripts/Installers/Install-Toolset.ps1
@@ -48,7 +48,8 @@ $ErrorActionPreference = "Stop"
Import-Module -Name ImageHelpers -Force
# Get toolcache content from toolset
-$tools = Get-ToolsetContent | Select-Object -ExpandProperty toolcache
+$ToolsToInstall = @("Python", "Node")
+$tools = Get-ToolsetContent | Select-Object -ExpandProperty toolcache | Where {$ToolsToInstall -contains $_.Name}
foreach ($tool in $tools) {
# Get versions manifest for current tool
diff --git a/images/win/scripts/Installers/Validate-ToolCache.ps1 b/images/win/scripts/Installers/Validate-ToolCache.ps1
index 526254e2b..f2e900aeb 100644
--- a/images/win/scripts/Installers/Validate-ToolCache.ps1
+++ b/images/win/scripts/Installers/Validate-ToolCache.ps1
@@ -177,10 +177,6 @@ function ToolcacheTest {
Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $markdownDescription
}
-# PyPy test
-$PyPyTests = @("python.exe", "bin\pip.exe")
-ToolcacheTest -SoftwareName "PyPy" -ExecTests $PyPyTests
-
# Ruby test
$RubyTests = @("bin\ruby.exe")
ToolcacheTest -SoftwareName "Ruby" -ExecTests $RubyTests
diff --git a/images/win/scripts/Installers/Validate-Toolset.ps1 b/images/win/scripts/Installers/Validate-Toolset.ps1
index 34c613d93..048256c35 100644
--- a/images/win/scripts/Installers/Validate-Toolset.ps1
+++ b/images/win/scripts/Installers/Validate-Toolset.ps1
@@ -63,8 +63,9 @@ Import-Module -Name ImageHelpers -Force
# Define executables for cached tools
$toolsExecutables = @{
- Python = @("python.exe", "Scripts\pip.exe");
+ Python = @("python.exe", "Scripts\pip.exe")
node = @("node.exe", "npm")
+ PyPy = @("python.exe", "Scripts\pip.exe")
}
# Get toolcache content from toolset
@@ -78,6 +79,11 @@ foreach($tool in $tools) {
$toolExecs = $toolsExecutables[$tool.name]
foreach ($version in $tool.versions) {
+ # Add wildcard if missing
+ if (-not $version.Contains('*')) {
+ $version += '.*'
+ }
+
# Check if version folder exists
$expectedVersionPath = Join-Path $toolPath $version
if (-not (Test-Path $expectedVersionPath)) {
@@ -100,8 +106,14 @@ foreach($tool in $tools) {
Write-Host "Run validation test for $($tool.name)($($tool.arch)) $($foundVersion.name) executables..."
Run-ExecutableTests -Executables $toolExecs -ToolPath $foundVersionArchPath
+ $foundVersionName = $foundVersion.name
+ if ($tool.name -eq 'PyPy')
+ {
+ $pypyVersion = & "$foundVersionArchPath\python.exe" -c "import sys;print(sys.version.split('\n')[1])"
+ $foundVersionName = "{0} {1}" -f $foundVersionName, $pypyVersion
+ }
# Add to tool version to markdown
- $markdownDescription += "_Version:_ $($foundVersion.name)
"
+ $markdownDescription += "_Version:_ $foundVersionName
"
}
# Create markdown description for system default tool
diff --git a/images/win/toolcache-2016.json b/images/win/toolcache-2016.json
index 1b4f1a6c7..449be64fa 100644
--- a/images/win/toolcache-2016.json
+++ b/images/win/toolcache-2016.json
@@ -2,9 +2,6 @@
"@actions/toolcache-ruby-windows-x64": [
"2.4", "2.5", "2.6", "2.7"
],
- "@actions/toolcache-pypy-windows-x86": [
- "2", "3"
- ],
"@actions/toolcache-boost-windows-msvc-14.1-x32-x64": [
"1.69", "1.72"
]
diff --git a/images/win/toolcache-2019.json b/images/win/toolcache-2019.json
index 6873c14e5..354f518f6 100644
--- a/images/win/toolcache-2019.json
+++ b/images/win/toolcache-2019.json
@@ -2,9 +2,6 @@
"@actions/toolcache-ruby-windows-x64": [
"2.4", "2.5", "2.6", "2.7"
],
- "@actions/toolcache-pypy-windows-x86": [
- "2", "3"
- ],
"@actions/toolcache-boost-windows-msvc-14.1-x32-x64": [
"1.69"
],
diff --git a/images/win/toolset-2016.json b/images/win/toolset-2016.json
index c1500a4a1..9eb2be40a 100644
--- a/images/win/toolset-2016.json
+++ b/images/win/toolset-2016.json
@@ -27,6 +27,15 @@
"3.8.*"
]
},
+ {
+ "name": "PyPy",
+ "arch": "x86",
+ "platform" : "win32",
+ "versions": [
+ "2.7",
+ "3.6"
+ ]
+ },
{
"name": "node",
"url" : "https://raw.githubusercontent.com/actions/node-versions/master/versions-manifest.json",
diff --git a/images/win/toolset-2019.json b/images/win/toolset-2019.json
index c1500a4a1..9eb2be40a 100644
--- a/images/win/toolset-2019.json
+++ b/images/win/toolset-2019.json
@@ -27,6 +27,15 @@
"3.8.*"
]
},
+ {
+ "name": "PyPy",
+ "arch": "x86",
+ "platform" : "win32",
+ "versions": [
+ "2.7",
+ "3.6"
+ ]
+ },
{
"name": "node",
"url" : "https://raw.githubusercontent.com/actions/node-versions/master/versions-manifest.json",