Install PyPy from official site instead of NPM for Ubuntu (#879)

* install PyPy from official site

images/linux/scripts/installers/Install-Toolset.ps1

@@ -28,7 +28,8 @@ $ErrorActionPreference = "Stop"
 
 # Get toolset content
 $toolsetJson = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw
-$tools = ConvertFrom-Json -InputObject $toolsetJson | Select-Object -ExpandProperty toolcache
+$toolsToInstall = @("Python", "Node")
+$tools = ConvertFrom-Json -InputObject $toolsetJson | Select-Object -ExpandProperty toolcache | Where {$ToolsToInstall -contains $_.Name}
 
 foreach ($tool in $tools) {
     # Get versions manifest for current tool

images/linux/scripts/installers/Validate-Toolset.ps1

@@ -27,8 +27,9 @@ $ErrorActionPreference = "Stop"
 
 # Define executables for cached tools
 $toolsExecutables = @{
-    Python = @("python", "bin/pip");
+    Python = @("python", "bin/pip")
     node = @("bin/node", "bin/npm")
+    PyPy = @("bin/python", "bin/pip")
 }
 
 # Get toolset content
@@ -43,6 +44,11 @@ foreach($tool in $tools) {
     $toolExecs = $toolsExecutables[$tool.name]
 
     foreach ($version in $tool.versions) {
+        # Add wildcard if missing
+        if (-not $version.Contains('*')) {
+            $version += '.*'
+        }
+
         # Check if version folder exists
         $expectedVersionPath = Join-Path $toolPath $version
         if (-not (Test-Path $expectedVersionPath)) {
@@ -59,7 +65,14 @@ foreach($tool in $tools) {
         Write-Host "Run validation test for $($tool.name)($($tool.arch)) $($foundVersion.name) executables..."
         Run-ExecutableTests -Executables $toolExecs -ToolPath $foundVersionPath
 
+        $foundVersionName = $foundVersion.name
+        if ($tool.name -eq 'PyPy')
+        {
+            $pypyVersion = & "$foundVersionPath/bin/python" -c "import sys;print(sys.version.split('\n')[1])"
+            $foundVersionName = "{0} {1}" -f $foundVersionName, $pypyVersion
+        }
+
         # Add tool version to documentation
-        Invoke-Expression "bash -c `"source $env:HELPER_SCRIPTS/document.sh; DocumentInstalledItemIndent '$($tool.name) $($foundVersion.name)'`""
+        Invoke-Expression "bash -c `"source $env:HELPER_SCRIPTS/document.sh; DocumentInstalledItemIndent '$($tool.name) $foundVersionName'`""
     }
 }

images/linux/scripts/installers/hosted-tool-cache.sh

@@ -54,9 +54,3 @@ rubys=$(ls $AGENT_TOOLSDIRECTORY/Ruby)
 for ruby in $rubys; do
 	DocumentInstalledItemIndent "Ruby $ruby"
 done;
-
-DocumentInstalledItem "PyPy:"
-pypys=$(ls $AGENT_TOOLSDIRECTORY/PyPy)
-for pypy in $pypys; do
-	DocumentInstalledItemIndent "PyPy $pypy"
-done;

images/linux/scripts/installers/pypy.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+################################################################################
+##  File:  pypy.sh
+##  Desc:  Installs PyPy
+################################################################################
+
+# Source the helpers for use with the script
+source $HELPER_SCRIPTS/document.sh
+
+# Fail out if any setups fail
+set -e
+
+# This function installs PyPy using the specified arguments:
+#   $1=PACKAGE_URL
+function InstallPyPy
+{
+    PACKAGE_URL=$1
+
+    PACKAGE_TAR_NAME=$(echo $PACKAGE_URL | awk -F/ '{print $NF}')
+    echo "Downloading tar archive '$PACKAGE_TAR_NAME' - '$PACKAGE_URL'"
+    PACKAGE_TAR_TEMP_PATH="/tmp/$PACKAGE_TAR_NAME"
+    wget -q -O $PACKAGE_TAR_TEMP_PATH $PACKAGE_URL
+
+    echo "Expand '$PACKAGE_TAR_NAME' to the /tmp folder"
+    tar xf $PACKAGE_TAR_TEMP_PATH -C /tmp
+
+    # Get Python version
+    PACKAGE_NAME=${PACKAGE_TAR_NAME/.tar.bz2/}
+    MAJOR_VERSION=$(echo ${PACKAGE_NAME/pypy/} | cut -d. -f1)
+    PYTHON_MAJOR="python$MAJOR_VERSION"
+
+    if [ $MAJOR_VERSION != 2 ]; then
+        PYPY_MAJOR="pypy$MAJOR_VERSION"
+    else
+        PYPY_MAJOR="pypy"
+    fi
+
+    PACKAGE_TEMP_FOLDER="/tmp/$PACKAGE_NAME"
+    PYTHON_FULL_VERSION=$("$PACKAGE_TEMP_FOLDER/bin/$PYPY_MAJOR" -c "import sys;print('{}.{}.{}'.format(sys.version_info[0],sys.version_info[1],sys.version_info[2]))")
+
+    # PyPy folder structure
+    PYPY_TOOLCACHE_PATH=$AGENT_TOOLSDIRECTORY/PyPy
+    PYPY_TOOLCACHE_VERSION_PATH=$PYPY_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
+    PYPY_TOOLCACHE_VERSION_ARCH_PATH=$PYPY_TOOLCACHE_VERSION_PATH/x64
+
+    echo "Check if PyPy hostedtoolcache folder exist..."
+    if [ ! -d $PYPY_TOOLCACHE_PATH ]; then
+        mkdir -p $PYPY_TOOLCACHE_PATH
+    fi
+
+    echo "Create PyPy '$PYPY_TOOLCACHE_VERSION_PATH' folder"
+    mkdir $PYPY_TOOLCACHE_VERSION_PATH
+
+    echo "Move PyPy '$PACKAGE_TEMP_FOLDER' binaries to '$PYPY_TOOLCACHE_VERSION_ARCH_PATH' folder"
+    mv $PACKAGE_TEMP_FOLDER $PYPY_TOOLCACHE_VERSION_ARCH_PATH
+
+    echo "Create additional symlinks (Required for UsePythonVersion Azure DevOps task)"
+    cd $PYPY_TOOLCACHE_VERSION_ARCH_PATH/bin
+    ln -s $PYPY_MAJOR $PYTHON_MAJOR
+    ln -s $PYTHON_MAJOR python
+
+    chmod +x ./python ./$PYTHON_MAJOR
+
+    echo "Install latest Pip"
+    ./python -m ensurepip
+    ./python -m pip install --ignore-installed pip
+
+    echo "Create complete file"
+    touch $PYPY_TOOLCACHE_VERSION_PATH/x64.complete
+
+    echo "Remove '$PACKAGE_TAR_TEMP_PATH'"
+    rm -f $PACKAGE_TAR_TEMP_PATH
+}
+
+function getPyPyVersions
+{
+    uri="https://api.bitbucket.org/2.0/repositories/pypy/pypy/downloads?pagelen=100"
+    curl -s -N $uri | jq -r ".values[].links.self.href|select(contains(\"linux64\"))"
+}
+
+# Installation PyPy
+pypyVersions=$(getPyPyVersions)
+toolsetJson="$INSTALLER_SCRIPT_FOLDER/toolset.json"
+toolsetVersions=$(cat $toolsetJson | jq -r '.toolcache[] | select(.name | contains("PyPy")) | .versions[]')
+
+for toolsetVersion in $toolsetVersions; do
+    latestMajorPyPyVersion=$(echo "${pypyVersions}" | grep -E "pypy${toolsetVersion}-v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+-" | head -1)
+
+    if [[ -z "$latestMajorPyPyVersion" ]]; then
+        echo "Failed to get PyPy version '$toolsetVersion'"
+        exit 1
+    fi
+
+    InstallPyPy $latestMajorPyPyVersion
+done
+
+chown -R "$SUDO_USER:$SUDO_USER" "$AGENT_TOOLSDIRECTORY/PyPy"

images/linux/scripts/installers/test-toolcache.sh

@@ -70,4 +70,3 @@ done;
 AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache
 
 Test_Hostedtoolcache_Tool "Ruby" "x64/bin/ruby -e 'puts RUBY_VERSION' | egrep -o '[0-9]+\.[0-9]+'"
-Test_Hostedtoolcache_Tool "PyPy" "x64/bin/python -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+' | cut -d '.' -f 1"