Delete builders/nix-python-builder.psm1

* Included ubuntu-arm64 in  Build Python package yml

* Updated builder and installer scripts

* Updated test scripts

* Modified test file

* Replaced macos-11 with macos-13

* Update build-python-packages.yml

.github/workflows/build-python-packages.yml

@@ -1,11 +1,11 @@
 name: Build Python package
-run-name: Generate Python ${{ inputs.VERSION || '3.11.0' }}
+run-name: Generate Python ${{ inputs.VERSION || '3.12.3' }}
 on:
   workflow_dispatch:
     inputs:
       VERSION:
         description: 'Python version to build and upload'
-        default: '3.11.0'
+        default: '3.12.3'
         required: true
       PUBLISH_RELEASES:
         description: 'Whether to publish releases'
@@ -15,7 +15,7 @@ on:
       PLATFORMS:
         description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
         required: true
-        default: 'ubuntu-18.04,ubuntu-20.04,ubuntu-22.04,macos-11,windows-2019_x64,windows-2019_x86'
+        default: 'ubuntu-20.04,ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,macos-11_x64,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-2019_arm64'
   pull_request:
     paths-ignore:
     - 'versions-manifest.json'
@@ -25,7 +25,7 @@ on:
     - 'main'
 
 env:
-  VERSION: ${{ inputs.VERSION || '3.11.0' }}
+  VERSION: ${{ inputs.VERSION || '3.12.3' }}
 defaults:
   run:
     shell: pwsh
@@ -39,26 +39,35 @@ jobs:
       - name: Generate execution matrix
         id: generate-matrix
         run: |
-          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-18.04,ubuntu-20.04,ubuntu-22.04,macos-11,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
+          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,macos-11,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-2019_arm64' }}".Split(",").Trim()
           $matrix = @()
 
           foreach ($configuration in $configurations) {
             $parts = $configuration.Split("_")
             $os = $parts[0]
             $arch = if ($parts[1]) {$parts[1]} else {"x64"}
+
             switch -wildcard ($os) {
               "*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
               "*macos*" { $platform = 'darwin' }
               "*windows*" { $platform = 'win32' }
             }
+
+            if ($configuration -eq "ubuntu-22.04_arm64") {
+              $os = "setup-actions-ubuntu-arm64-2-core"
+            }elseif ($configuration -eq "windows-2019_arm64") {
+              $os = "setup-actions-windows-arm64-4-core"
+            }
             $matrix += @{
               'platform' = $platform
               'os' = $os
               'arch' = $arch
+              
+
             }
           }
           echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT
-
+          
   build_python:
     needs: generate_matrix
     strategy:
@@ -67,24 +76,47 @@ jobs:
         include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
     runs-on: ${{ matrix.os }}
     env:
-      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.12.3' }}-${{ matrix.platform }}-${{ matrix.arch }}
     steps:
 
+      - name: Setup Environment on Windows ARM64 Runner
+        if: matrix.os == 'setup-actions-windows-arm64-4-core'
+        shell: powershell
+        run: |
+          # Install Chocolatey
+          Set-ExecutionPolicy Bypass -Scope Process -Force
+          [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
+          iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
+          echo "C:\ProgramData\Chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+      
+          # Install PowerShell
+          choco install powershell-core -y
+          echo "C:\Program Files\PowerShell\7" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+      
+          # Install Git
+          choco install git -y
+          echo "C:\Program Files\Git\cmd" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+      
+          
+           # Install 7-Zip
+          choco install 7zip -y
+          echo "C:\ProgramData\chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
       - name: Check out repository code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
 
+    
       - name: Build Python ${{ env.VERSION }}
         run: |
           ./builders/build-python.ps1 -Version $env:VERSION `
                   -Platform ${{ matrix.platform }} -Architecture ${{ matrix.arch }}
-
       - name: Publish artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ env.ARTIFACT_NAME }}
           path: ${{ runner.temp }}/artifact
+          if-no-files-found: error
 
   test_python:
     needs: [generate_matrix, build_python]
@@ -94,22 +126,52 @@ jobs:
         include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
     runs-on: ${{ matrix.os }}
     env:
-      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.12.3' }}-${{ matrix.platform }}-${{ matrix.arch }}
     steps:
-
+    
+      - name: Setup Environment on Windows ARM64 Runner
+        if: matrix.os == 'setup-actions-windows-arm64-4-core'
+        shell: powershell
+        run: |
+          # Install Chocolatey
+          Set-ExecutionPolicy Bypass -Scope Process -Force
+          [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
+          iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
+          echo "C:\ProgramData\Chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+      
+          # Install PowerShell
+          choco install powershell-core -y
+          echo "C:\Program Files\PowerShell\7" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+      
+          # Install Git
+          choco install git -y
+          echo "C:\Program Files\Git\cmd" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+      
+          
+           # Install 7-Zip
+          choco install 7zip -y
+          echo "C:\ProgramData\chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+          
       - name: Check out repository code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
 
       - name: Fully cleanup the toolcache directory before testing
         run: ./helpers/clean-toolcache.ps1 -ToolName "Python"
 
-      - name: Download artifact
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ runner.temp }}
+      - name: Delete macOS /Library/Frameworks/Python.framework
+        if: matrix.platform == 'darwin'
+        shell: bash
+        run: if [ -d /Library/Frameworks/Python.framework ]; then sudo rm -rf /Library/Frameworks/Python.framework; fi
 
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
+
+    
       - name: Extract files
         run: |
           if ('${{ matrix.platform }}' -eq 'win32') {
@@ -123,6 +185,7 @@ jobs:
 
       - name: Apply build artifact to the local machine
         run: |
+          $PSNativeCommandUseErrorActionPreference = $true
           if ('${{ matrix.platform }}' -eq 'win32') {
             powershell ./setup.ps1
             } else {
@@ -132,7 +195,7 @@ jobs:
         working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
 
       - name: Setup Python ${{ env.VERSION }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.VERSION }}
           architecture: ${{ matrix.arch }}
@@ -147,14 +210,14 @@ jobs:
         run: |
           $pythonLocation = which python
           if ('${{ matrix.platform }}' -eq 'darwin') { otool -L $pythonLocation } else { ldd $pythonLocation }
-
       - name: Run tests
         run: |
-            Install-Module Pester -Force -Scope CurrentUser
+            Install-Module Pester -Force -Scope CurrentUser -SkipPublisherCheck
             Import-Module Pester
             $pesterContainer = New-PesterContainer -Path './python-tests.ps1' -Data @{
                 Version="${{ env.VERSION }}";
                 Platform="${{ matrix.platform }}";
+                Architecture="${{ matrix.arch }}";
             }
             $Result = Invoke-Pester -Container $pesterContainer -PassThru
             if ($Result.FailedCount -gt 0) {
@@ -169,7 +232,7 @@ jobs:
       needs: test_python
       runs-on: ubuntu-latest
       steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
 
       - name: Publish Release ${{ env.VERSION }}
         id: create_release
@@ -181,22 +244,34 @@ jobs:
           release_name: ${{ env.VERSION }}
           body: |
             Python ${{ env.VERSION }}
-
+      - name: Generate hash for packages
+        run: |
+          $childItems = Get-Childitem -Path '.'
+          $childItems | Foreach-Object {
+            $packageObj = Get-Childitem -Path $_.FullName | Select-Object -First 1
+            Write-Host "Package: $($packageObj.Name)"
+            $actualHash = (Get-FileHash -Path $packageObj.FullName -Algorithm sha256).Hash
+            $hashString = "$actualHash $($packageObj.Name)"
+            Write-Host "$hashString"
+            Add-Content -Path ./hashes.sha256 -Value "$hashString"
+          } 
+  
       - name: Upload release assets
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const fs = require('fs');
             for (let artifactDir of fs.readdirSync('.')) {
-              let artifactName = fs.readdirSync(`${artifactDir}`)[0];
+              let artifactName = fs.lstatSync(artifactDir).isDirectory() ? fs.readdirSync(`${artifactDir}`)[0] : artifactDir;
+  
               console.log(`Upload ${artifactName} asset`);
               github.rest.repos.uploadReleaseAsset({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 release_id: ${{ steps.create_release.outputs.id }},
                 name: artifactName,
-                data: fs.readFileSync(`./${artifactDir}/${artifactName}`)
+                data: fs.lstatSync(artifactDir).isDirectory() ? fs.readFileSync(`./${artifactDir}/${artifactName}`) : fs.readFileSync(`./${artifactName}`).toString()
               });
             }
 
@@ -206,7 +281,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Trigger "Create Pull Request" workflow
-      uses: actions/github-script@v6
+      uses: actions/github-script@v7
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
         script: |
@@ -216,4 +291,3 @@ jobs:
             workflow_id: 'create-pr.yml',
             ref: 'main'
           });
-

.github/workflows/manifest-config-validation.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: true
 

.github/workflows/releases-validation.yml

@@ -14,11 +14,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-18.04, ubuntu-20.04]
-        python: [3.5.4, 3.6.7, 3.7.5, 3.8.1]
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04]
+        python: [3.9.13, 3.10.11, 3.11.8]
     steps:
     - name: setup-python ${{ matrix.python }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python }}
 

.github/workflows/test-python-version.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-18.04, ubuntu-20.04] 
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04] 
     steps:
     - name: Setup Python ${{ github.event.inputs.version }}
       uses: actions/setup-python@main

README.md

@@ -21,5 +21,7 @@ We aim to make new versions of Python available as soon as they are released. Bo
 
 When a new version of an operating system is released and made available for use with [GitHub hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources), we will provide the **latest existing patch versions of Python for all major versions that have not reached [end-of-life](https://devguide.python.org/versions/)**.  Any subsequent Python versions will be made available for the new OS as well.
 
+Beginning **approximately six months prior** to the removal of a Python version from the [versions-manifest.json](https://github.com/actions/python-versions/blob/main/versions-manifest.json) file, a pinned issue will be created in the [setup-python](https://github.com/actions/setup-python) repository. This pinned issue will provide important details about the upcoming end of support, including the specific date, as well as any other notes, relevant updates or alternatives. We encourage users to regularly check pinned issues for updates on tool versions they are using for maximum transparency, security, performance and overall compatibility with their projects.
+
 ## Contribution
 Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure

builders/macos-python-builder.psm1

@@ -7,7 +7,7 @@ class macOSPythonBuilder : NixPythonBuilder {
 
     .DESCRIPTION
     Contains methods that required to build macOS Python artifact from sources. Inherited from base NixPythonBuilder.
-    
+
     While python.org provides precompiled binaries for macOS, switching to them risks breaking existing customers.
     If we wanted to start using the official binaries instead of building from source, we should avoid changing previous versions
     so we remain backwards compatible.
@@ -31,6 +31,14 @@ class macOSPythonBuilder : NixPythonBuilder {
         .SYNOPSIS
         Prepare system environment by installing dependencies and required packages.
         #>
+        
+        if ($this.Version -eq "3.7.17") {
+            # We have preinstalled ncurses and readLine on the hoster runners. But we need to install bzip2 for 
+            # setting up an environemnt 
+            # If we get any issues realted to ncurses or readline we can try to run this command
+            # brew install ncurses readline
+            Execute-Command -Command "brew install bzip2"
+        }
     }
 
     [void] Configure() {
@@ -67,14 +75,19 @@ class macOSPythonBuilder : NixPythonBuilder {
 
             # For Python 3.7.2 and 3.7.3 we need to provide PATH for zlib to pack it properly. Otherwise the build will fail
             # with the error: zipimport.ZipImportError: can't decompress data; zlib not available
-            if ($this.Version -eq "3.7.2" -or $this.Version -eq "3.7.3") {
+            if ($this.Version -eq "3.7.2" -or $this.Version -eq "3.7.3" -or $this.Version -eq "3.7.17") {
                 $env:LDFLAGS = "-L/usr/local/opt/zlib/lib"
                 $env:CFLAGS = "-I/usr/local/opt/zlib/include"
             }
 
             if ($this.Version -gt "3.7.12") {
                 $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'"
-	    }
+	        }
+
+            if ($this.Version -eq "3.7.17") {
+                $env:LDFLAGS += " -L$(brew --prefix bzip2)/lib -L$(brew --prefix readline)/lib -L$(brew --prefix ncurses)/lib"
+                $env:CFLAGS += " -I$(brew --prefix bzip2)/include -I$(brew --prefix readline)/include -I$(brew --prefix ncurses)/include"
+            }
         }
 
         ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
@@ -151,6 +164,7 @@ class macOSPythonBuilder : NixPythonBuilder {
         $variablesToReplace = @{
             "{{__VERSION_FULL__}}" = $this.Version;
             "{{__PKG_NAME__}}" = $this.GetPkgName();
+            "{{__ARCH__}}" = $this.Architecture;
         }
 
         $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
@@ -166,7 +180,7 @@ class macOSPythonBuilder : NixPythonBuilder {
 
         $PkgVersion = [semver]"3.11.0-beta.1"
 
-        if ($this.Version -ge $PkgVersion) {
+        if (($this.Version -ge $PkgVersion) -or ($this.Architecture -eq "arm64")) {
             Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..."
             $this.DownloadPkg()
 

builders/nix-python-builder.psm1

@@ -1,168 +0,0 @@
-using module "./python-builder.psm1"
-
-class NixPythonBuilder : PythonBuilder {
-    <#
-    .SYNOPSIS
-    Base Python builder class for *Nix systems.
-
-    .DESCRIPTION
-    Contains methods that required to build Python artifact for *nix systems. Inherited from base PythonBuilder class.
-
-    .PARAMETER version
-    The version of Python that should be built.
-
-    .PARAMETER Platform
-    The type of platform for which Python should be built.
-
-    .PARAMETER PlatformVersion
-    The version of platform for which Python should be built.
-
-    .PARAMETER InstallationTemplateName
-    The name of template that will be used to create installation script for generated Python artifact.
-
-    .PARAMETER InstallationScriptName	
-    The name of installation script that will be generated for Python artifact.
-
-    .PARAMETER OutputArtifactName
-    The name of archive with Python binaries that will be generated as part of Python artifact. 
-
-    #>
-
-    [string] $InstallationTemplateName
-    [string] $InstallationScriptName
-    [string] $OutputArtifactName
-
-    NixPythonBuilder(
-        [semver] $version,
-        [string] $architecture,
-        [string] $platform
-    ) : Base($version, $architecture, $platform) {
-        $this.InstallationTemplateName = "nix-setup-template.sh"	
-        $this.InstallationScriptName = "setup.sh"
-        $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.tar.gz"
-    }
-
-    [uri] GetSourceUri() {
-        <#
-        .SYNOPSIS
-        Get base Python URI and return complete URI for Python sources.
-        #>
-
-        $base = $this.GetBaseUri()
-        $versionName = $this.GetBaseVersion()
-        $nativeVersion = Convert-Version -version $this.Version
-
-        return "${base}/${versionName}/Python-${nativeVersion}.tgz"
-    }
-
-    [string] GetPythonBinary() {
-        <#
-        .SYNOPSIS
-        Return name of Python binary.
-        #>
-
-        if ($this.Version.Major -eq 2) { $pythonBinary = "python" } else { $pythonBinary = "python3" }
-        return $pythonBinary
-    }
-
-    [string] Download() {
-        <#
-        .SYNOPSIS
-        Download Python sources and extract them at temporary work folder. Returns expanded archive location path.
-        #>
-
-        $sourceUri = $this.GetSourceUri()
-        Write-Host "Sources URI: $sourceUri"
-
-        $archiveFilepath = Download-File -Uri $sourceUri -OutputFolder $this.WorkFolderLocation
-        $expandedSourceLocation = Join-Path -Path $this.TempFolderLocation -ChildPath "SourceCode"
-        New-Item -Path $expandedSourceLocation -ItemType Directory
-
-        Extract-TarArchive -ArchivePath $archiveFilepath -OutputDirectory $expandedSourceLocation
-        Write-Debug "Done; Sources location: $expandedSourceLocation"
-
-        return $expandedSourceLocation
-    }
-
-    [void] CreateInstallationScript() {
-        <#
-        .SYNOPSIS
-        Create Python artifact installation script based on template specified in InstallationTemplateName property.
-        #>
-
-        $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName -ItemType File
-        $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath $this.InstallationTemplateName
-
-        $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
-
-        $variablesToReplace = @{
-            "{{__VERSION_FULL__}}" = $this.Version;
-        }
-        $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
-
-        $installationTemplateContent | Out-File -FilePath $installationScriptLocation
-
-        Write-Debug "Done; Installation script location: $installationScriptLocation)"
-    }
-
-    [void] Make() {
-        <#
-        .SYNOPSIS
-        Executes "make" and "make install" commands for configured build sources. Make output will be writen in build_output.txt located in artifact location folder.
-        #>
-
-        Write-Debug "make Python $($this.Version)-$($this.Architecture) $($this.Platform)"
-        $buildOutputLocation = New-Item -Path $this.WorkFolderLocation -Name "build_output.txt" -ItemType File
-        
-        Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue	
-        Execute-Command -Command "make install" -ErrorAction Continue
-
-        Write-Debug "Done; Make log location: $buildOutputLocation"
-    }
-
-    [void] CopyBuildResults() {
-        $buildFolder = $this.GetFullPythonToolcacheLocation()
-        Move-Item -Path "$buildFolder/*" -Destination $this.WorkFolderLocation
-    }
-
-    [void] ArchiveArtifact() {
-        $OutputPath = Join-Path $this.ArtifactFolderLocation $this.OutputArtifactName
-        Create-TarArchive -SourceFolder $this.WorkFolderLocation -ArchivePath $OutputPath
-    }
-
-    [void] Build() {
-        <#
-        .SYNOPSIS
-        Build Python artifact from sources. 
-        #>
-
-        Write-Host "Prepare Python Hostedtoolcache location..."
-        $this.PreparePythonToolcacheLocation()
-
-        Write-Host "Prepare system environment..."
-        $this.PrepareEnvironment()
-
-        Write-Host "Download Python $($this.Version)[$($this.Architecture)] sources..."
-        $sourcesLocation = $this.Download()
-
-        Push-Location -Path $sourcesLocation
-        Write-Host "Configure for $($this.Platform)..."
-        $this.Configure()
-
-        Write-Host "Make for $($this.Platform)..."
-        $this.Make()
-        Pop-Location
-
-        Write-Host "Generate structure dump"
-        New-ToolStructureDump -ToolPath $this.GetFullPythonToolcacheLocation() -OutputFolder $this.WorkFolderLocation
-
-        Write-Host "Copying build results to destination location"
-        $this.CopyBuildResults()
-
-        Write-Host "Create installation script..."
-        $this.CreateInstallationScript()
-
-        Write-Host "Archive artifact..."
-        $this.ArchiveArtifact()
-    }
-}

builders/ubuntu-python-builder.psm1

@@ -37,16 +37,9 @@ class UbuntuPythonBuilder : NixPythonBuilder {
         $configureString += " --enable-shared"
         $configureString += " --enable-optimizations"
 
-        ### Compile with ucs4 for Python 2.x. On 3.x, ucs4 is enabled by default
-        if ($this.Version -lt "3.0.0") {
-            $configureString += " --enable-unicode=ucs4"
-        }
-
-        ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
+        ### Compile with support of loadable sqlite extensions.
         ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
-        if ($this.Version -ge "3.2.0") {
-            $configureString += " --enable-loadable-sqlite-extensions"
-        }
+        $configureString += " --enable-loadable-sqlite-extensions"
 
         Write-Host "The passed configure options are: "
         Write-Host $configureString
@@ -60,18 +53,15 @@ class UbuntuPythonBuilder : NixPythonBuilder {
         Prepare system environment by installing dependencies and required packages.
         #>
 
-        if (($this.Version -gt "3.0.0") -and ($this.Version -lt "3.5.3")) {
-            Write-Host "Python3 versions lower than 3.5.3 are not supported"
+        if ($this.Version -lt "3.5.3") {
+            Write-Host "Python versions lower than 3.5.3 are not supported"
             exit 1
         }
 
         ### Compile with tkinter support
-        if ($this.Version -gt "3.0.0") {
-            $tkinterInstallString = "sudo apt-get install -y --allow-downgrades python3-tk tk-dev"
-        } else {
-            $tkinterInstallString = "sudo apt install -y python-tk tk-dev"
-        }
+        $tkinterInstallString = "sudo apt-get install -y --allow-downgrades python3-tk tk-dev"
 
+        Execute-Command -Command "sudo apt-get update"
         Execute-Command -Command $tkinterInstallString
 
         ### Install dependent packages
@@ -84,7 +74,8 @@ class UbuntuPythonBuilder : NixPythonBuilder {
             "libsqlite3-dev",
             "libncursesw5-dev",
             "libreadline-dev",
-            "libgdbm-dev"
+            "libgdbm-dev",
+            "liblzma-dev"
         ) | ForEach-Object {
             Execute-Command -Command "sudo apt install -y $_"
         }

builders/win-python-builder.psm1

@@ -1,3 +1,4 @@
+
 using module "./python-builder.psm1"
 
 class WinPythonBuilder : PythonBuilder {
@@ -60,8 +61,11 @@ class WinPythonBuilder : PythonBuilder {
             } else {
                 $ArchitectureExtension = ".amd64"
             }
+        }elseif ($this.Architecture -eq "arm64") {
+                $ArchitectureExtension = "-arm64"
         }
 
+
         return $ArchitectureExtension
     }
 

config/python-manifest-config.json

@@ -1,5 +1,5 @@
 {
-    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?(x\\d+)",
+    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+)",
     "groups": {
         "arch": 4,
         "platform": 2,

installers/macos-pkg-setup-template.sh

@@ -2,6 +2,7 @@ set -e
 
 PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
 PYTHON_PKG_NAME="{{__PKG_NAME__}}"
+ARCH="{{__ARCH__}}"
 MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
 MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
 
@@ -18,7 +19,7 @@ fi
 
 PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
 PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
-PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
 PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
 PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
 
@@ -29,10 +30,10 @@ if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
 else
     # remove ALL other directories for same major.minor python versions
     find $PYTHON_TOOLCACHE_PATH -name "${MAJOR_VERSION}.${MINOR_VERSION}.*"|while read python_version;do
-        python_version_x64="$python_version/x64"
-        if [ -e "$python_version_x64" ];then
-            echo "Deleting Python $python_version_x64"
-            rm -rf "$python_version_x64"
+        python_version_arch="$python_version/$ARCH"
+        if [ -e "$python_version_arch" ];then
+            echo "Deleting Python $python_version_arch"
+            rm -rf "$python_version_arch"
         fi
     done
 fi
@@ -55,7 +56,7 @@ ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
 
 cd bin/
 
-# This symlink already exists if Python version with the same major.minor version is installed, 
+# This symlink already exists if Python version with the same major.minor version is installed,
 # since we do not remove the framework folder
 if [ ! -f $PYTHON_MAJOR_MINOR ]; then
     ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
@@ -68,11 +69,12 @@ fi
 chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
 
 echo "Upgrading pip..."
+export PIP_ROOT_USER_ACTION=ignore
 ./python -m ensurepip
-./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location
+./python -m pip install --upgrade --force-reinstall pip --disable-pip-version-check --no-warn-script-location
 
 echo "Install OpenSSL certificates"
 sh -e "${PYTHON_APPLICATION_PATH}/Install Certificates.command"
 
 echo "Create complete file"
-touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete
+touch $PYTHON_TOOLCACHE_VERSION_PATH/${ARCH}.complete

installers/nix-setup-template.sh

@@ -1,6 +1,7 @@
 set -e
 
 PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
+ARCH="{{__ARCH__}}"
 MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
 MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
 
@@ -17,7 +18,7 @@ fi
 
 PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
 PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
-PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
 
 echo "Check if Python hostedtoolcache folder exist..."
 if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
@@ -49,8 +50,9 @@ fi
 chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR python
 
 echo "Upgrading pip..."
+export PIP_ROOT_USER_ACTION=ignore
 ./python -m ensurepip
-./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location
+./python -m pip install --upgrade --force-reinstall pip --disable-pip-version-check --no-warn-script-location
 
 echo "Create complete file"
-touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete
+touch $PYTHON_TOOLCACHE_VERSION_PATH/$ARCH.complete

installers/win-setup-template.ps1

@@ -101,7 +101,9 @@ if ($null -ne $InstalledVersions) {
         if (Test-Path -Path $InstalledVersion) {
             Write-Host "Deleting $InstalledVersion..."
             Remove-Item -Path $InstalledVersion -Recurse -Force
-            Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose
+            if (Test-Path -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete") {
+                Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose
+            }
         }
     }
 } else {
@@ -131,8 +133,12 @@ if ($MajorVersion -ne "2") {
 }
 
 Write-Host "Install and upgrade Pip"
+$Env:PIP_ROOT_USER_ACTION = "ignore"
 $PythonExePath = Join-Path -Path $PythonArchPath -ChildPath "python.exe"
-cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip --no-warn-script-location"
+cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade --force-reinstall pip --no-warn-script-location"
+if ($LASTEXITCODE -ne 0) {
+    Throw "Error happened during pip installation / upgrade"
+}
 
 Write-Host "Create complete file"
 New-Item -ItemType File -Path $PythonVersionPath -Name "$Architecture.complete" | Out-Null

tests/ManifestConfig.Tests.ps1

@@ -5,16 +5,16 @@ $Configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
 
 $stableTestCases = @(
     @{ ReleaseName = "python-3.8.3-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
-    @{ ReleaseName = "python-3.8.3-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
 ) | ForEach-Object { $_.Configuration = $Configuration; $_ }
 
 $unstableTestCases = @(
     @{ ReleaseName = "python-3.9.0-alpha.2-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
-    @{ ReleaseName = "python-3.9.0-rc.4-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-rc.4-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
 ) | ForEach-Object { $_.Configuration = $Configuration; $_ }

tests/python-tests.ps1

@@ -2,7 +2,9 @@ param (
     [semver] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
     $Version,
     [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
-    $Platform
+    $Platform,
+    [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    $Architecture
 )
 
 Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
@@ -56,7 +58,7 @@ Describe "Tests" {
     #     }
     # }
 
-    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0")) {
+    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($Architecture -ne "arm64"))) {
         It "Check if sqlite3 module is installed" {
             "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
         }
@@ -80,7 +82,7 @@ Describe "Tests" {
 
         It "Check if python configuration is correct" {
             $nativeVersion = Convert-Version -version $Version
-            "python ./sources/python-config-test.py $Version $nativeVersion" | Should -ReturnZeroExitCode
+            "python ./sources/python-config-test.py $Version $nativeVersion $Architecture" | Should -ReturnZeroExitCode
         }
 
         It "Check if shared libraries are linked correctly" {
@@ -101,4 +103,8 @@ Describe "Tests" {
     It "Check urlopen with HTTPS works" {
         "python ./sources/python-urlopen-https.py" | Should -ReturnZeroExitCode
     }
+
+    It "Check a single dist-info per distribution is present" {
+        "python ./sources/dist-info.py" | Should -ReturnZeroExitCode
+    }
 }

tests/sources/dist-info.py

@@ -0,0 +1,24 @@
+import glob
+import os.path
+import sysconfig
+from collections import defaultdict
+
+
+def check_dist_info():
+    paths = set([sysconfig.get_path("purelib"), sysconfig.get_path("platlib")])
+    versions = defaultdict(list)
+    for path in paths:
+        pattern = os.path.join(path, "*.dist-info")
+        for dist_info in glob.glob(pattern):
+            name = os.path.basename(dist_info).split("-", maxsplit=1)[0]
+            versions[name].append(dist_info)
+    exit_code = 0
+    for name in versions:
+        if len(versions[name]) > 1:
+            print("multiple dist-info found for {}: {}".format(name, versions[name]))
+            exit_code = 1
+    exit(exit_code)
+
+
+if __name__ == "__main__":
+    check_dist_info()

tests/sources/python-config-output.py

@@ -1,6 +1,4 @@
-import distutils.sysconfig
 import sysconfig
 
 from pprint import pprint
 pprint(sysconfig.get_config_vars())
-pprint(distutils.sysconfig.get_config_vars())

tests/sources/python-config-test.py

@@ -1,5 +1,3 @@
-import distutils.sysconfig
-from distutils.version import LooseVersion
 import sysconfig
 import sys
 import platform
@@ -9,12 +7,13 @@ import os
 os_type = platform.system()
 version = sys.argv[1]
 nativeVersion = sys.argv[2]
+architecture = sys.argv[3]
 
 versions=version.split(".")
 version_major=int(versions[0])
 version_minor=int(versions[1])
 
-pkg_installer = os_type == 'Darwin' and (version_major == 3 and version_minor >= 11)
+pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (architecture == "arm64"))
 
 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
@@ -29,7 +28,7 @@ if os_type == 'Darwin': expected_ld_library_extension = 'dylib'
 if pkg_installer:
     expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib'
 else:
-    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/x64/lib'
+    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/{architecture}/lib'
 
 # Check modules
 ### Validate libraries path
@@ -40,7 +39,7 @@ if lib_dir_path != expected_lib_dir_path:
 ### Validate shared libraries
 if is_shared:
     print('%s was built with shared extensions' % ld_library_name)
-    
+
     ### Validate libpython extension
     ld_library_extension = ld_library_name.split('.')[-1]
     if ld_library_extension != expected_ld_library_extension:
@@ -54,7 +53,7 @@ else:
 ### Validate macOS
 if os_type == 'Darwin':
     ### Validate openssl links
-    if LooseVersion(nativeVersion) < LooseVersion("3.7.0"):
+    if version_major == 3 and version_minor < 7:
         expected_ldflags = '-L/usr/local/opt/openssl@1.1/lib'
         ldflags = sysconfig.get_config_var('LDFLAGS')
 
@@ -64,7 +63,7 @@ if os_type == 'Darwin':
     else:
         expected_openssl_includes = '-I/usr/local/opt/openssl@1.1/include'
         expected_openssl_ldflags ='-L/usr/local/opt/openssl@1.1/lib'
-        
+
         openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES')
         openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS')
 
@@ -81,4 +80,4 @@ if os_type == 'Darwin':
 if sys.version_info < (3, 12):
     if not have_libreadline:
         print('Missing libreadline')
-        exit(1)
+        exit(1)

tests/sources/python-modules.py

@@ -7,7 +7,6 @@ from __future__ import print_function
 
 import importlib
 import sys
-import platform
 
 # The Python standard library as of Python 3.0
 standard_library = [
@@ -266,17 +265,36 @@ if sys.version_info >= (3, 10):
 if sys.version_info >= (3, 11):
     standard_library.remove('binhex')
 
-# Exclude tkinter and turtle for Python 3.11 alpha temporarily
-if sys.version_info >= (3, 11) and platform.system() == 'Linux' and '18.04' in platform.version():
-    standard_library.remove('tkinter')
-    standard_library.remove('turtle')
-
 # 'smtpd', 'asyncore' and 'asynchat' modules have been removed from Python 3.12
+# https://docs.python.org/dev/whatsnew/3.12.html
 if sys.version_info >= (3, 12):
+    standard_library.remove('distutils')
+    standard_library.remove('imp')
     standard_library.remove('smtpd')
     standard_library.remove('asyncore')
     standard_library.remove('asynchat')
 
+# 'aifc', 'cgi', 'cgitb', 'chunk', 'crypt', 'imghdr', 'lib2to3', 'mailcap', 'nntplib',
+# 'pipes', 'sndhdr', 'sunau', 'telnetlib', 'uu' and 'xdrlib' modules have been removed
+# from Python 3.13
+# https://docs.python.org/dev/whatsnew/3.13.html
+if sys.version_info >= (3, 13):
+    standard_library.remove('aifc')
+    standard_library.remove('cgi')
+    standard_library.remove('cgitb')
+    standard_library.remove('chunk')
+    standard_library.remove('crypt')
+    standard_library.remove('imghdr')
+    standard_library.remove('lib2to3')
+    standard_library.remove('mailcap')
+    standard_library.remove('nntplib')
+    standard_library.remove('pipes')
+    standard_library.remove('sndhdr')
+    standard_library.remove('sunau')
+    standard_library.remove('telnetlib')
+    standard_library.remove('uu')
+    standard_library.remove('xdrlib')
+
 # Remove tkinter and Easter eggs
 excluded_modules = [
     'antigravity',