Update versions-manifest (#228)

Co-authored-by: Service account <no-reply@microsoft.com>

.github/workflows/build-python-packages.yml

@@ -15,7 +15,7 @@ on:
       PLATFORMS:
         description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
         required: true
-        default: 'ubuntu-18.04,ubuntu-20.04,ubuntu-22.04,macos-11,windows-2019_x64,windows-2019_x86'
+        default: 'ubuntu-20.04,ubuntu-22.04,macos-11_x64,macos-11_arm64,windows-2019_x64,windows-2019_x86'
   pull_request:
     paths-ignore:
     - 'versions-manifest.json'
@@ -39,7 +39,7 @@ jobs:
       - name: Generate execution matrix
         id: generate-matrix
         run: |
-          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-18.04,ubuntu-20.04,ubuntu-22.04,macos-11,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
+          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,macos-11,macos-11_arm64,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
           $matrix = @()
 
           foreach ($configuration in $configurations) {
@@ -155,6 +155,7 @@ jobs:
             $pesterContainer = New-PesterContainer -Path './python-tests.ps1' -Data @{
                 Version="${{ env.VERSION }}";
                 Platform="${{ matrix.platform }}";
+                Architecture="${{ matrix.arch }}";
             }
             $Result = Invoke-Pester -Container $pesterContainer -PassThru
             if ($Result.FailedCount -gt 0) {

.github/workflows/releases-validation.yml

@@ -14,8 +14,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-18.04, ubuntu-20.04]
-        python: [3.5.4, 3.6.7, 3.7.5, 3.8.1]
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04]
+        python: [3.9.12, 3.10.8, 3.11.10]
     steps:
     - name: setup-python ${{ matrix.python }}
       uses: actions/setup-python@v4

.github/workflows/test-python-version.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-18.04, ubuntu-20.04] 
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04] 
     steps:
     - name: Setup Python ${{ github.event.inputs.version }}
       uses: actions/setup-python@main

builders/macos-python-builder.psm1

@@ -7,7 +7,7 @@ class macOSPythonBuilder : NixPythonBuilder {
 
     .DESCRIPTION
     Contains methods that required to build macOS Python artifact from sources. Inherited from base NixPythonBuilder.
-    
+
     While python.org provides precompiled binaries for macOS, switching to them risks breaking existing customers.
     If we wanted to start using the official binaries instead of building from source, we should avoid changing previous versions
     so we remain backwards compatible.
@@ -151,6 +151,7 @@ class macOSPythonBuilder : NixPythonBuilder {
         $variablesToReplace = @{
             "{{__VERSION_FULL__}}" = $this.Version;
             "{{__PKG_NAME__}}" = $this.GetPkgName();
+            "{{__ARCH__}}" = $this.Architecture;
         }
 
         $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
@@ -166,7 +167,7 @@ class macOSPythonBuilder : NixPythonBuilder {
 
         $PkgVersion = [semver]"3.11.0-beta.1"
 
-        if ($this.Version -ge $PkgVersion) {
+        if (($this.Version -ge $PkgVersion) -or ($this.Architecture -eq "arm64")) {
             Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..."
             $this.DownloadPkg()
 

config/python-manifest-config.json

@@ -1,5 +1,5 @@
 {
-    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?(x\\d+)",
+    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+)",
     "groups": {
         "arch": 4,
         "platform": 2,

installers/macos-pkg-setup-template.sh

@@ -2,6 +2,7 @@ set -e
 
 PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
 PYTHON_PKG_NAME="{{__PKG_NAME__}}"
+ARCH="{{__ARCH__}}"
 MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
 MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
 
@@ -18,7 +19,7 @@ fi
 
 PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
 PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
-PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
 PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
 PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
 
@@ -29,10 +30,10 @@ if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
 else
     # remove ALL other directories for same major.minor python versions
     find $PYTHON_TOOLCACHE_PATH -name "${MAJOR_VERSION}.${MINOR_VERSION}.*"|while read python_version;do
-        python_version_x64="$python_version/x64"
-        if [ -e "$python_version_x64" ];then
-            echo "Deleting Python $python_version_x64"
-            rm -rf "$python_version_x64"
+        python_version_arch="$python_version/$ARCH"
+        if [ -e "$python_version_arch" ];then
+            echo "Deleting Python $python_version_arch"
+            rm -rf "$python_version_arch"
         fi
     done
 fi
@@ -55,7 +56,7 @@ ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
 
 cd bin/
 
-# This symlink already exists if Python version with the same major.minor version is installed, 
+# This symlink already exists if Python version with the same major.minor version is installed,
 # since we do not remove the framework folder
 if [ ! -f $PYTHON_MAJOR_MINOR ]; then
     ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
@@ -75,4 +76,4 @@ echo "Install OpenSSL certificates"
 sh -e "${PYTHON_APPLICATION_PATH}/Install Certificates.command"
 
 echo "Create complete file"
-touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete
+touch $PYTHON_TOOLCACHE_VERSION_PATH/${ARCH}.complete

tests/ManifestConfig.Tests.ps1

@@ -5,16 +5,16 @@ $Configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
 
 $stableTestCases = @(
     @{ ReleaseName = "python-3.8.3-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
-    @{ ReleaseName = "python-3.8.3-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
 ) | ForEach-Object { $_.Configuration = $Configuration; $_ }
 
 $unstableTestCases = @(
     @{ ReleaseName = "python-3.9.0-alpha.2-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
-    @{ ReleaseName = "python-3.9.0-rc.4-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-rc.4-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
 ) | ForEach-Object { $_.Configuration = $Configuration; $_ }

tests/python-tests.ps1

@@ -2,7 +2,9 @@ param (
     [semver] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
     $Version,
     [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
-    $Platform
+    $Platform,
+    [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    $Architecture
 )
 
 Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
@@ -56,7 +58,7 @@ Describe "Tests" {
     #     }
     # }
 
-    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0")) {
+    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($Architecture -ne "arm64"))) {
         It "Check if sqlite3 module is installed" {
             "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
         }
@@ -80,7 +82,7 @@ Describe "Tests" {
 
         It "Check if python configuration is correct" {
             $nativeVersion = Convert-Version -version $Version
-            "python ./sources/python-config-test.py $Version $nativeVersion" | Should -ReturnZeroExitCode
+            "python ./sources/python-config-test.py $Version $nativeVersion $Architecture" | Should -ReturnZeroExitCode
         }
 
         It "Check if shared libraries are linked correctly" {

tests/sources/python-config-output.py

@@ -1,6 +1,4 @@
-import distutils.sysconfig
 import sysconfig
 
 from pprint import pprint
 pprint(sysconfig.get_config_vars())
-pprint(distutils.sysconfig.get_config_vars())

tests/sources/python-config-test.py

@@ -1,5 +1,3 @@
-import distutils.sysconfig
-from distutils.version import LooseVersion
 import sysconfig
 import sys
 import platform
@@ -9,12 +7,13 @@ import os
 os_type = platform.system()
 version = sys.argv[1]
 nativeVersion = sys.argv[2]
+architecture = sys.argv[3]
 
 versions=version.split(".")
 version_major=int(versions[0])
 version_minor=int(versions[1])
 
-pkg_installer = os_type == 'Darwin' and (version_major == 3 and version_minor >= 11)
+pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (architecture == "arm64"))
 
 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
@@ -40,7 +39,7 @@ if lib_dir_path != expected_lib_dir_path:
 ### Validate shared libraries
 if is_shared:
     print('%s was built with shared extensions' % ld_library_name)
-    
+
     ### Validate libpython extension
     ld_library_extension = ld_library_name.split('.')[-1]
     if ld_library_extension != expected_ld_library_extension:
@@ -54,7 +53,7 @@ else:
 ### Validate macOS
 if os_type == 'Darwin':
     ### Validate openssl links
-    if LooseVersion(nativeVersion) < LooseVersion("3.7.0"):
+    if version_major == 3 and version_minor < 7:
         expected_ldflags = '-L/usr/local/opt/openssl@1.1/lib'
         ldflags = sysconfig.get_config_var('LDFLAGS')
 
@@ -64,7 +63,7 @@ if os_type == 'Darwin':
     else:
         expected_openssl_includes = '-I/usr/local/opt/openssl@1.1/include'
         expected_openssl_ldflags ='-L/usr/local/opt/openssl@1.1/lib'
-        
+
         openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES')
         openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS')
 
@@ -81,4 +80,4 @@ if os_type == 'Darwin':
 if sys.version_info < (3, 12):
     if not have_libreadline:
         print('Missing libreadline')
-        exit(1)
+        exit(1)

tests/sources/python-modules.py

@@ -7,7 +7,6 @@ from __future__ import print_function
 
 import importlib
 import sys
-import platform
 
 # The Python standard library as of Python 3.0
 standard_library = [
@@ -266,13 +265,11 @@ if sys.version_info >= (3, 10):
 if sys.version_info >= (3, 11):
     standard_library.remove('binhex')
 
-# Exclude tkinter and turtle for Python 3.11 alpha temporarily
-if sys.version_info >= (3, 11) and platform.system() == 'Linux' and '18.04' in platform.version():
-    standard_library.remove('tkinter')
-    standard_library.remove('turtle')
-
 # 'smtpd', 'asyncore' and 'asynchat' modules have been removed from Python 3.12
+# https://docs.python.org/dev/whatsnew/3.12.html
 if sys.version_info >= (3, 12):
+    standard_library.remove('distutils')
+    standard_library.remove('imp')
     standard_library.remove('smtpd')
     standard_library.remove('asyncore')
     standard_library.remove('asynchat')

versions-manifest.json

@@ -1,4 +1,94 @@
 [
+  {
+    "version": "3.12.0-beta.1",
+    "stable": false,
+    "release_url": "https://github.com/actions/python-versions/releases/tag/3.12.0-beta.1-5076755295",
+    "files": [
+      {
+        "filename": "python-3.12.0-beta.1-darwin-arm64.tar.gz",
+        "arch": "arm64",
+        "platform": "darwin",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.12.0-beta.1-5076755295/python-3.12.0-beta.1-darwin-arm64.tar.gz"
+      },
+      {
+        "filename": "python-3.12.0-beta.1-darwin-x64.tar.gz",
+        "arch": "x64",
+        "platform": "darwin",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.12.0-beta.1-5076755295/python-3.12.0-beta.1-darwin-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.12.0-beta.1-linux-20.04-x64.tar.gz",
+        "arch": "x64",
+        "platform": "linux",
+        "platform_version": "20.04",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.12.0-beta.1-5076755295/python-3.12.0-beta.1-linux-20.04-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.12.0-beta.1-linux-22.04-x64.tar.gz",
+        "arch": "x64",
+        "platform": "linux",
+        "platform_version": "22.04",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.12.0-beta.1-5076755295/python-3.12.0-beta.1-linux-22.04-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.12.0-beta.1-win32-x64.zip",
+        "arch": "x64",
+        "platform": "win32",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.12.0-beta.1-5076755295/python-3.12.0-beta.1-win32-x64.zip"
+      },
+      {
+        "filename": "python-3.12.0-beta.1-win32-x86.zip",
+        "arch": "x86",
+        "platform": "win32",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.12.0-beta.1-5076755295/python-3.12.0-beta.1-win32-x86.zip"
+      }
+    ]
+  },
+  {
+    "version": "3.12.0-alpha.7",
+    "stable": false,
+    "release_url": "https://github.com/actions/python-versions/releases/tag/3.12.0-alpha.7-4618531320",
+    "files": [
+      {
+        "filename": "python-3.12.0-alpha.7-darwin-arm64.tar.gz",
+        "arch": "arm64",
+        "platform": "darwin",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.12.0-alpha.7-4618531320/python-3.12.0-alpha.7-darwin-arm64.tar.gz"
+      },
+      {
+        "filename": "python-3.12.0-alpha.7-darwin-x64.tar.gz",
+        "arch": "x64",
+        "platform": "darwin",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.12.0-alpha.7-4618531320/python-3.12.0-alpha.7-darwin-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.12.0-alpha.7-linux-20.04-x64.tar.gz",
+        "arch": "x64",
+        "platform": "linux",
+        "platform_version": "20.04",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.12.0-alpha.7-4618531320/python-3.12.0-alpha.7-linux-20.04-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.12.0-alpha.7-linux-22.04-x64.tar.gz",
+        "arch": "x64",
+        "platform": "linux",
+        "platform_version": "22.04",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.12.0-alpha.7-4618531320/python-3.12.0-alpha.7-linux-22.04-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.12.0-alpha.7-win32-x64.zip",
+        "arch": "x64",
+        "platform": "win32",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.12.0-alpha.7-4618531320/python-3.12.0-alpha.7-win32-x64.zip"
+      },
+      {
+        "filename": "python-3.12.0-alpha.7-win32-x86.zip",
+        "arch": "x86",
+        "platform": "win32",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.12.0-alpha.7-4618531320/python-3.12.0-alpha.7-win32-x86.zip"
+      }
+    ]
+  },
   {
     "version": "3.12.0-alpha.6",
     "stable": false,
@@ -275,6 +365,51 @@
       }
     ]
   },
+  {
+    "version": "3.11.3",
+    "stable": true,
+    "release_url": "https://github.com/actions/python-versions/releases/tag/3.11.3-4626642838",
+    "files": [
+      {
+        "filename": "python-3.11.3-darwin-arm64.tar.gz",
+        "arch": "arm64",
+        "platform": "darwin",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.11.3-4626642838/python-3.11.3-darwin-arm64.tar.gz"
+      },
+      {
+        "filename": "python-3.11.3-darwin-x64.tar.gz",
+        "arch": "x64",
+        "platform": "darwin",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.11.3-4626642838/python-3.11.3-darwin-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.11.3-linux-20.04-x64.tar.gz",
+        "arch": "x64",
+        "platform": "linux",
+        "platform_version": "20.04",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.11.3-4626642838/python-3.11.3-linux-20.04-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.11.3-linux-22.04-x64.tar.gz",
+        "arch": "x64",
+        "platform": "linux",
+        "platform_version": "22.04",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.11.3-4626642838/python-3.11.3-linux-22.04-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.11.3-win32-x64.zip",
+        "arch": "x64",
+        "platform": "win32",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.11.3-4626642838/python-3.11.3-win32-x64.zip"
+      },
+      {
+        "filename": "python-3.11.3-win32-x86.zip",
+        "arch": "x86",
+        "platform": "win32",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.11.3-4626642838/python-3.11.3-win32-x86.zip"
+      }
+    ]
+  },
   {
     "version": "3.11.2",
     "stable": true,
@@ -1008,6 +1143,51 @@
       }
     ]
   },
+  {
+    "version": "3.10.11",
+    "stable": true,
+    "release_url": "https://github.com/actions/python-versions/releases/tag/3.10.11-4626646535",
+    "files": [
+      {
+        "filename": "python-3.10.11-darwin-arm64.tar.gz",
+        "arch": "arm64",
+        "platform": "darwin",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.10.11-4626646535/python-3.10.11-darwin-arm64.tar.gz"
+      },
+      {
+        "filename": "python-3.10.11-darwin-x64.tar.gz",
+        "arch": "x64",
+        "platform": "darwin",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.10.11-4626646535/python-3.10.11-darwin-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.10.11-linux-20.04-x64.tar.gz",
+        "arch": "x64",
+        "platform": "linux",
+        "platform_version": "20.04",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.10.11-4626646535/python-3.10.11-linux-20.04-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.10.11-linux-22.04-x64.tar.gz",
+        "arch": "x64",
+        "platform": "linux",
+        "platform_version": "22.04",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.10.11-4626646535/python-3.10.11-linux-22.04-x64.tar.gz"
+      },
+      {
+        "filename": "python-3.10.11-win32-x64.zip",
+        "arch": "x64",
+        "platform": "win32",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.10.11-4626646535/python-3.10.11-win32-x64.zip"
+      },
+      {
+        "filename": "python-3.10.11-win32-x86.zip",
+        "arch": "x86",
+        "platform": "win32",
+        "download_url": "https://github.com/actions/python-versions/releases/download/3.10.11-4626646535/python-3.10.11-win32-x86.zip"
+      }
+    ]
+  },
   {
     "version": "3.10.10",
     "stable": true,