Update versions-manifest (#190)

Co-authored-by: Service account <no-reply@microsoft.com>

builders/macos-python-builder.psm1

@@ -83,4 +83,92 @@ class macOSPythonBuilder : NixPythonBuilder {
 
         Execute-Command -Command $configureString
     }
+
+    [string] GetPkgName() {
+        <#
+        .SYNOPSIS
+        Return Python installation Package.
+        #>
+
+        $nativeVersion = Convert-Version -version $this.Version
+        $architecture = "-macos11"
+        $extension = ".pkg"
+
+        $pkg = "python-${nativeVersion}${architecture}${extension}"
+
+        return $pkg
+    }
+
+    [uri] GetPkgUri() {
+        <#
+        .SYNOPSIS
+        Get base Python URI and return complete URI for Python installation package.
+        #>
+
+        $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $pkg = $this.GetPkgName()
+
+        $uri = "${base}/${versionName}/${pkg}"
+
+        return $uri
+    }
+
+    [string] DownloadPkg() {
+        <#
+        .SYNOPSIS
+        Download Python installation executable into artifact location.
+        #>
+
+        $pkgUri = $this.GetPkgUri()
+
+        Write-Host "Sources URI: $pkgUri"
+        $pkgLocation = Download-File -Uri $pkgUri -OutputFolder $this.WorkFolderLocation
+        Write-Debug "Done; Package location: $pkgLocation"
+
+        New-Item -Path $this.WorkFolderLocation -Name "build_output.txt"  -ItemType File
+        return $pkgLocation
+    }
+
+    [void] CreateInstallationScriptPkg() {
+        <#
+        .SYNOPSIS
+        Create Python artifact installation script based on specified template.
+        #>
+
+        $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath "macos-pkg-setup-template.sh"
+        $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
+        $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName  -ItemType File
+
+        $variablesToReplace = @{
+            "{{__VERSION_FULL__}}" = $this.Version;
+            "{{__PKG_NAME__}}" = $this.GetPkgName();
+        }
+
+        $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
+        $installationTemplateContent | Out-File -FilePath $installationScriptLocation
+        Write-Debug "Done; Installation script location: $installationScriptLocation)"
+    }
+
+    [void] Build() {
+        <#
+        .SYNOPSIS
+        Generates Python artifact from downloaded Python installation executable.
+        #>
+
+        $PkgVersion = [semver]"3.11.0-beta.1"
+
+        if ($this.Version -ge $PkgVersion) {
+            Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..."
+            $this.DownloadPkg()
+
+            Write-Host "Create installation script..."
+            $this.CreateInstallationScriptPkg()
+        } else {
+            ([NixPythonBuilder]$this).Build()
+        }
+
+        Write-Host "Archive artifact"
+        $this.ArchiveArtifact()
+    }
 }

installers/macos-pkg-setup-template.sh

@@ -0,0 +1,72 @@
+set -e
+
+PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
+PYTHON_PKG_NAME="{{__PKG_NAME__}}"
+MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
+MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
+
+PYTHON_MAJOR=python$MAJOR_VERSION
+PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION
+PYTHON_MAJOR_MINOR=python$MAJOR_VERSION$MINOR_VERSION
+
+if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then
+    # No AGENT_TOOLSDIRECTORY on GitHub images
+    TOOLCACHE_ROOT=$RUNNER_TOOL_CACHE
+else
+    TOOLCACHE_ROOT=$AGENT_TOOLSDIRECTORY
+fi
+
+PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
+PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64
+PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
+PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
+
+echo "Check if Python hostedtoolcache folder exist..."
+if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
+    echo "Creating Python hostedtoolcache folder..."
+    mkdir -p $PYTHON_TOOLCACHE_PATH
+else
+    # remove ALL other directories for same major.minor python versions
+    find $PYTHON_TOOLCACHE_PATH -name "${MAJOR_VERSION}.${MINOR_VERSION}.*"|while read python_version;do
+        python_version_x64="$python_version/x64"
+        if [ -e "$python_version_x64" ];then
+            echo "Deleting Python $python_version_x64"
+            rm -rf "$python_version_x64"
+        fi
+    done
+fi
+
+echo "Install Python binaries from prebuilt package"
+sudo installer -pkg $PYTHON_PKG_NAME -target /
+
+echo "Create hostedtoolcach symlinks (Required for the backward compatibility)"
+echo "Create Python $PYTHON_FULL_VERSION folder"
+mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+
+ln -s "${PYTHON_FRAMEWORK_PATH}/bin" bin
+ln -s "${PYTHON_FRAMEWORK_PATH}/include" include
+ln -s "${PYTHON_FRAMEWORK_PATH}/share" share
+ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib
+
+echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
+ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
+
+cd bin/
+ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
+if [ ! -f python ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR python
+fi
+
+chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
+
+echo "Upgrading pip..."
+./python -m ensurepip
+./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location
+
+echo "Install OpenSSL certificates"
+sh -e "${PYTHON_APPLICATION_PATH}/Install Certificates.command"
+
+echo "Create complete file"
+touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete

tests/python-tests.ps1

@@ -96,4 +96,8 @@ Describe "Tests" {
             "./dist/simple-test" | Should -ReturnZeroExitCode
         }
     }
+
+    It "Check urlopen with HTTPS works" {
+        "python ./sources/python-urlopen-https.py" | Should -ReturnZeroExitCode
+    }
 }

tests/sources/python-config-test.py

@@ -10,6 +10,12 @@ os_type = platform.system()
 version = sys.argv[1]
 nativeVersion = sys.argv[2]
 
+versions=version.split(".")
+version_major=int(versions[0])
+version_minor=int(versions[1])
+
+pkg_installer = os_type == 'Darwin' and (version_major == 3 and version_minor >= 11)
+
 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
 
@@ -19,7 +25,11 @@ have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE")
 ### Define expected variables
 if os_type == 'Linux': expected_ld_library_extension = 'so'
 if os_type == 'Darwin': expected_ld_library_extension = 'dylib'
-expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version)
+
+if pkg_installer:
+    expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib'
+else:
+    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/x64/lib'
 
 # Check modules
 ### Validate libraries path
@@ -38,7 +48,8 @@ if is_shared:
         exit(1)
 else:
     print('%s was built without shared extensions' % ld_library_name)
-    exit(1)
+    if not pkg_installer:
+        exit(1)
 
 ### Validate macOS
 if os_type == 'Darwin':
@@ -59,12 +70,14 @@ if os_type == 'Darwin':
 
         if openssl_includes != expected_openssl_includes:
             print('Invalid openssl_includes: %s; Expected: %s' % (openssl_includes, expected_openssl_includes))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
         if openssl_ldflags != expected_openssl_ldflags:
             print('Invalid openssl_ldflags: %s; Expected: %s' % (openssl_ldflags, expected_openssl_ldflags))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
 
 ### Validate libreadline
 if not have_libreadline:
     print('Missing libreadline')
-    exit(1)
+    exit(1)

tests/sources/python-urlopen-https.py

@@ -0,0 +1,10 @@
+import sys
+
+if sys.version_info[0] == 2:
+    from urllib2 import urlopen
+else:
+    from urllib.request import urlopen
+
+response = urlopen("https://raw.githubusercontent.com/actions/python-versions/c641695f6a07526c18f10e374e503e649fef9427/.gitignore")
+data = response.read()
+assert len(data) == 140, len(data)

versions-manifest.json

@@ -1,4 +1,50 @@
 [
+  {
+    "version": "3.11.0-rc.2",
+    "stable": false,
+    "release_url": "https://github.com/actions/python-versions/releases/tag/3.11.0-rc.2-3172748327",
+    "files": [
+      {
+        "filename": "python-3.11.0-rc.2-darwin-x64.tar.gz",
+        "arch": "x64",
+        "platform": "darwin",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.11.0-rc.2-3172748327/python-3.11.0-rc.2-darwin-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.11.0-rc.2-linux-18.04-x64.tar.gz",
+        "arch": "x64",
+        "platform": "linux",
+        "platform_version": "18.04",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.11.0-rc.2-3172748327/python-3.11.0-rc.2-linux-18.04-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.11.0-rc.2-linux-20.04-x64.tar.gz",
+        "arch": "x64",
+        "platform": "linux",
+        "platform_version": "20.04",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.11.0-rc.2-3172748327/python-3.11.0-rc.2-linux-20.04-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.11.0-rc.2-linux-22.04-x64.tar.gz",
+        "arch": "x64",
+        "platform": "linux",
+        "platform_version": "22.04",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.11.0-rc.2-3172748327/python-3.11.0-rc.2-linux-22.04-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.11.0-rc.2-win32-x64.zip",
+        "arch": "x64",
+        "platform": "win32",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.11.0-rc.2-3172748327/python-3.11.0-rc.2-win32-x64.zip"
+      },
+      {
+        "filename": "python-3.11.0-rc.2-win32-x86.zip",
+        "arch": "x86",
+        "platform": "win32",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.11.0-rc.2-3172748327/python-3.11.0-rc.2-win32-x86.zip"
+      }
+    ]
+  },
   {
     "version": "3.11.0-rc.1",
     "stable": false,