Update versions-manifest (#344)

Co-authored-by: Service account <no-reply@microsoft.com>

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @actions/setup-actions-team

.github/workflows/build-python-packages.yml

@@ -0,0 +1,261 @@
+name: Build Python package
+run-name: Generate Python ${{ inputs.VERSION || '3.12.3' }}
+on:
+  workflow_dispatch:
+    inputs:
+      VERSION:
+        description: 'Python version to build and upload'
+        default: '3.12.3'
+        required: true
+      PUBLISH_RELEASES:
+        description: 'Whether to publish releases'
+        required: true
+        type: boolean
+        default: false
+      THREADING_BUILD_MODES:
+        description: 'CPython threading build modes'
+        required: true
+        type: string
+        default: 'default,freethreaded'
+      PLATFORMS:
+        description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
+        required: true
+        default: 'ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,ubuntu-24.04_arm64,macos-13_x64,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-11_arm64'
+  pull_request:
+    paths-ignore:
+    - 'versions-manifest.json'
+    - 'LICENSE'
+    - '**.md'
+    branches:
+    - 'main'
+
+env:
+  VERSION: ${{ inputs.VERSION || '3.12.3' }}
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  generate_matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generate-matrix.outputs.matrix }}
+    steps:
+      - name: Generate execution matrix
+        id: generate-matrix
+        run: |
+          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,ubuntu-24.04_arm64,macos-13,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-11_arm64' }}".Split(",").Trim()
+          [String[]]$buildModes = "${{ inputs.threading_build_modes || 'default' }}".Split(",").Trim()
+          $matrix = @()
+          
+          foreach ($configuration in $configurations) {
+            foreach ($buildMode in $buildModes) {
+              $parts = $configuration.Split("_")
+              $os = $parts[0]
+              $arch = if ($parts[1]) {$parts[1]} else {"x64"}
+              switch -wildcard ($os) {
+                "*ubuntu*" { $platform = $os.Replace("ubuntu","linux"); if ($arch -eq "arm64" ) { $os = "${os}-arm" } }
+                "*macos*" { $platform = 'darwin' }
+                "*windows*" { $platform = 'win32'; if ($arch -eq "arm64" ) { $os = "${os}-arm" } }
+              }
+
+              if ($buildMode -eq "freethreaded") {
+                if ([semver]"${{ inputs.VERSION }}" -lt [semver]"3.13.0") {
+                  continue;
+                }
+                $arch += "-freethreaded"
+              }
+
+              $matrix += @{
+                'platform' = $platform
+                'os' = $os
+                'arch' = $arch
+              }
+            }
+          }
+          echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT
+          
+  build_python:
+    needs: generate_matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
+    runs-on: ${{ matrix.os }}
+    env:
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.12.3' }}-${{ matrix.platform }}-${{ matrix.arch }}
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Adding Symlink
+        if: matrix.os == 'macos-13'
+        run: ln -sf /usr/local/opt/tcl-tk@8 /usr/local/opt/tcl-tk
+
+      - name: Build Python ${{ env.VERSION }}
+        run: |
+          ./builders/build-python.ps1 -Version $env:VERSION `
+                  -Platform ${{ matrix.platform }} -Architecture ${{ matrix.arch }}
+                  
+      - name: Publish artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ${{ runner.temp }}/artifact
+          if-no-files-found: error
+
+  test_python:
+    needs: [generate_matrix, build_python]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
+    runs-on: ${{ matrix.os }}
+    env:
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.12.3' }}-${{ matrix.platform }}-${{ matrix.arch }}
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Fully cleanup the toolcache directory before testing
+        run: ./helpers/clean-toolcache.ps1 -ToolName "Python"
+
+      - name: Delete macOS /Library/Frameworks/Python.framework
+        if: matrix.platform == 'darwin'
+        shell: bash
+        run: if [ -d /Library/Frameworks/Python.framework ]; then sudo rm -rf /Library/Frameworks/Python.framework; fi
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
+
+      - name: Extract files
+        run: |
+          if ('${{ matrix.platform }}' -eq 'win32') {
+            $artifactName = "${{ env.ARTIFACT_NAME }}.zip"
+            7z.exe x "$artifactName" -y | Out-Null
+          } else {
+            $artifactName = "${{ env.ARTIFACT_NAME }}.tar.gz"
+            tar -xzf $artifactName
+          }
+        working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
+
+      - name: Apply build artifact to the local machine
+        run: |
+          $PSNativeCommandUseErrorActionPreference = $true
+          if ('${{ matrix.platform }}' -eq 'win32') {
+            powershell ./setup.ps1
+            } else {
+              sh ./setup.sh
+              cp ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}/build_output.txt ${{ runner.temp }}
+            }
+        working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
+
+      - name: Setup Python ${{ env.VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.VERSION }}
+          architecture: ${{ matrix.arch }}
+
+      - name: Python version
+        run: python -VVV
+
+      - name: Verbose sysconfig dump
+        if: runner.os == 'Linux' || runner.os == 'macOS'
+        run: python ./sources/python-config-output.py
+        working-directory: ${{ github.workspace }}/tests
+
+      - name: Verbose python binary links
+        if: runner.os == 'Linux' || runner.os == 'macOS'
+        run: |
+          $pythonLocation = which python
+          if ('${{ matrix.platform }}' -eq 'darwin') { otool -L $pythonLocation } else { ldd $pythonLocation }
+
+      - name: Run tests
+        run: |
+            Install-Module Pester -Force -Scope CurrentUser -SkipPublisherCheck
+            Import-Module Pester
+            $pesterContainer = New-PesterContainer -Path './python-tests.ps1' -Data @{
+                Version="${{ env.VERSION }}";
+                Platform="${{ matrix.platform }}";
+                Architecture="${{ matrix.arch }}";
+            }
+            $Result = Invoke-Pester -Container $pesterContainer -PassThru
+            if ($Result.FailedCount -gt 0) {
+              $host.SetShouldExit($Result.FailedCount)
+              exit $Result.FailedCount
+            }
+        working-directory: ${{ github.workspace }}/tests
+
+  publish_release:
+      name: Publish release
+      if: github.event_name == 'workflow_dispatch' && inputs.PUBLISH_RELEASES
+      needs: test_python
+      runs-on: ubuntu-latest
+      steps:
+      - uses: actions/download-artifact@v4
+
+      - name: Publish Release ${{ env.VERSION }}
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.VERSION }}-${{ github.run_id }}
+          release_name: ${{ env.VERSION }}
+          body: |
+            Python ${{ env.VERSION }}
+
+      - name: Generate hash for packages
+        run: |
+          $childItems = Get-Childitem -Path '.'
+          $childItems | Foreach-Object {
+            $packageObj = Get-Childitem -Path $_.FullName | Select-Object -First 1
+            Write-Host "Package: $($packageObj.Name)"
+            $actualHash = (Get-FileHash -Path $packageObj.FullName -Algorithm sha256).Hash
+            $hashString = "$actualHash $($packageObj.Name)"
+            Write-Host "$hashString"
+            Add-Content -Path ./hashes.sha256 -Value "$hashString"
+          } 
+  
+      - name: Upload release assets
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            for (let artifactDir of fs.readdirSync('.')) {
+              let artifactName = fs.lstatSync(artifactDir).isDirectory() ? fs.readdirSync(`${artifactDir}`)[0] : artifactDir;
+  
+              console.log(`Upload ${artifactName} asset`);
+              github.rest.repos.uploadReleaseAsset({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                release_id: ${{ steps.create_release.outputs.id }},
+                name: artifactName,
+                data: fs.lstatSync(artifactDir).isDirectory() ? fs.readFileSync(`./${artifactDir}/${artifactName}`) : fs.readFileSync(`./${artifactName}`).toString()
+              });
+            }
+
+  trigger_pr:
+    name: Trigger "Create Pull Request" workflow
+    needs: publish_release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Trigger "Create Pull Request" workflow
+      uses: actions/github-script@v7
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          github.rest.actions.createWorkflowDispatch({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            workflow_id: 'create-pr.yml',
+            ref: 'main'
+          });

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,17 @@
+name: CodeQL analysis
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 3 * * 0'
+
+jobs:
+  call-codeQL-analysis:
+    name: CodeQL analysis 
+    uses: actions/reusable-workflows/.github/workflows/codeql-analysis.yml@main
+    with:
+     languages: '["python"]'
+   

.github/workflows/create-pr.yml

@@ -0,0 +1,10 @@
+name: Create Pull Request
+on:
+  workflow_dispatch:
+
+jobs:
+  create-pr:
+    uses: actions/versions-package-tools/.github/workflows/create-pr-to-update-manifest.yml@main
+    with:
+      tool-name: "python"
+    secrets: inherit

.github/workflows/create-release.yml

@@ -0,0 +1,18 @@
+name: Create release
+on:
+  repository_dispatch:
+    types: [create-release]
+
+jobs:
+  create_release:
+    name: Create release ${{ github.event.client_payload.ToolVersion }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Create release for Python ${{ github.event.client_payload.ToolVersion }}
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ github.event.client_payload.TagName }}
+        release_name: ${{ github.event.client_payload.ToolVersion }}
+        body: ${{ github.event.client_payload.ReleaseBody }}

.github/workflows/get-python-versions.yml

@@ -0,0 +1,13 @@
+name: Get Python versions
+on:
+  schedule:
+    - cron: '0 3,15 * * *'
+  workflow_dispatch:
+
+jobs:
+  get-new-python-versions:
+    uses: actions/versions-package-tools/.github/workflows/get-new-tool-versions.yml@main
+    with:
+      tool-name: "Python"
+      image-url: "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+    secrets: inherit

.github/workflows/manifest-config-validation.yml

@@ -0,0 +1,32 @@
+name: Manifest config tests
+on: 
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - 'versions-manifest.json'
+
+jobs:
+  RunTests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        submodules: true
+
+    - name: Install Pester
+      shell: pwsh
+      run: |
+        Install-Module Pester -Force -Scope CurrentUser
+
+    - name: Run tests
+      shell: pwsh
+      run: |
+        Import-Module Pester
+        Invoke-Pester -Configuration @{
+          Run = @{
+            Path = "tests/ManifestConfig.Tests.ps1"
+            Exit = $true
+          }
+        }

.github/workflows/python-versions-runner.yml

@@ -0,0 +1,31 @@
+name: Python versions runner
+on:
+  workflow_dispatch:
+    inputs:
+      versions:
+        description: 'Versions to build'
+        required: true
+        default: '","'
+      publish-releases:
+        description: 'Whether to publish releases'
+        required: true
+        default: 'false'
+
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  trigger_builds:
+    name: Trigger python build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      
+      - name: Trigger python workflow
+        run: |
+          $versions = ${{ github.event.inputs.versions }}
+          ./builders/python-versions-runner.ps1 -Versions $versions.Split(",") -PublishRelease ${{ github.event.inputs.publish-releases }}
+        env:
+          TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/releases-validation.yml

@@ -0,0 +1,36 @@
+name: Validate 'versions-manifest.json' file
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'versions-manifest.json'
+
+jobs:
+  setup-versions-from-manifest:
+    name: Setup ${{ matrix.python }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-22.04, ubuntu-latest, macos-13]
+        python: [3.9.13, 3.10.11, 3.11.8, 3.12.7, 3.13.0]
+    steps:
+    - name: setup-python ${{ matrix.python }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Validate version
+      run: |
+        $pythonVersion = (python --version)
+        if ("Python ${{ matrix.python }}" -ne "$pythonVersion"){
+          Write-Host "The current version is $pythonVersion; expected version is ${{ matrix.python }}"
+          exit 1
+        }
+        $pythonVersion
+      shell: pwsh
+
+    - name: Run simple code
+      run: python -c 'import math; print(math.factorial(5))'

.github/workflows/test-python-version.yml

@@ -0,0 +1,34 @@
+name: Test Python version
+on: 
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Python version to build'
+        required: true
+      architecture:
+        description: 'The target architecture (x86, x64) of the Python'
+        required: false
+        default: 'x64'
+
+jobs:
+  test-python:
+    name: Test Python ${{ github.event.inputs.version }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-22.04, ubuntu-latest, macos-13] 
+    steps:
+    - name: Setup Python ${{ github.event.inputs.version }}
+      uses: actions/setup-python@main
+      with:
+        python-version: ${{ github.event.inputs.version }}
+        architecture: ${{ github.event.inputs.architecture }}
+
+    - name: Validate version
+      run: |
+        python --version
+      shell: pwsh
+
+    - name: Run simple code
+      run: python -c 'import math; print(math.factorial(5))'

.github/workflows/validate-manifest.yml

@@ -0,0 +1,21 @@
+name: Validate manifest
+on:
+# The GITHUB_TOKEN secret is used to create a PR
+# The pull_request event will not be triggered by it
+# That's one of the reasons we need the schedule to validate the versions-manifest.json file
+  schedule:
+    - cron: '0 8,20 * * *'
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'versions-manifest.json'
+
+jobs:
+  manifest:
+    uses: actions/versions-package-tools/.github/workflows/validate-manifest.yml@main
+    with:
+      tool-name: "Python"
+      image-url: "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+    secrets: inherit

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "helpers"]
 	path = helpers
 	url = https://github.com/actions/versions-package-tools
-	branch = master
+	branch = main

README.md

@@ -1,16 +1,27 @@
 # Python for Actions
-This repository contains the code and scripts that we use to build Python packages used in [virtual-environments](https://github.com/actions/virtual-environments) and accessible through the [setup-python](https://github.com/actions/setup-python) Action.  
-File [versions-manifest.json](./versions-manifest.json) contains the list of available and released versions.  
+This repository contains the code and scripts that we use to build Python packages used in [runner-images](https://github.com/actions/runner-images) and accessible through the [setup-python](https://github.com/actions/setup-python) Action.
+File [versions-manifest.json](./versions-manifest.json) contains the list of available and released versions.
 
-> Caution: this is prepared for and only permitted for use by actions `virtual-environments` and `setup-python` action.
+> Caution: this is prepared for and only permitted for use by actions `runner-images` and `setup-python` action.
 
-**Status**: Currently under development and in use for beta and preview actions.  This repo is undergoing rapid changes.
+Some versions are pre-installed on [runner-images](https://github.com/actions/runner-images) images.
+More versions are available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
 
-Some versions are pre-installed on [virtual-environments](https://github.com/actions/virtual-environments) images.  
-More versions will (soon!) be available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
+## Building installation packages
+**Ubuntu:** The official Python sources are downloaded from [python.org](https://www.python.org/ftp/python/), built using the `make` tool, and archived along with the installation script for further distribution and installation. We build each Python version against all [versions of Ubuntu](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) that are available at the time the Python version is released.
 
-## Adding new versions
-We are trying to build and release new versions of Python as soon as they are released. Please open an issue if any versions are missing.
+**macOS:** 
+- For **Python < 3.11**, the official Python sources are downloaded from [python.org](https://www.python.org/ftp/python/), built using the `make` tool, and archived along with the installation script for further distribution and installation.  For backward compatibility, we build each Python version against the oldest [version of macOS](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) that is available at the time the Python version is released.
+- For **Python >= 3.11**, the official macOS `universal2` Python binaries are simply downloaded from [python.org](https://www.python.org/ftp/python/) and archived along with the installation script for further distribution and installation.
+
+**Windows:** The official Python executables are simply downloaded from [python.org](https://www.python.org/ftp/python/) and archived along with the installation script for further distribution and installation.
+
+## Support policy
+We aim to make new versions of Python available as soon as they are released. Both stable and prerelease versions are considered for building and releasing. Please open an issue in [actions/setup-python](https://github.com/actions/setup-python) if a version you're looking for isn't available.
+
+When a new version of an operating system is released and made available for use with [GitHub hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources), we will provide the **latest existing patch versions of Python for all major versions that have not reached [end-of-life](https://devguide.python.org/versions/)**.  Any subsequent Python versions will be made available for the new OS as well.
+
+Beginning **approximately six months prior** to the removal of a Python version from the [versions-manifest.json](https://github.com/actions/python-versions/blob/main/versions-manifest.json) file, a pinned issue will be created in the [setup-python](https://github.com/actions/setup-python) repository. This pinned issue will provide important details about the upcoming end of support, including the specific date, as well as any other notes, relevant updates or alternatives. We encourage users to regularly check pinned issues for updates on tool versions they are using for maximum transparency, security, performance and overall compatibility with their projects.
 
 ## Contribution
 Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure

azure-pipelines/build-python-packages.yml

@@ -1,118 +0,0 @@
-name: $(date:yyyyMMdd)$(rev:.r)-Python-$(VERSION)
-trigger: none
-pr: none
-
-stages:
-- stage: Build_Python_MacOS
-  dependsOn: []
-  variables:
-    VmImage: 'macOS-10.14'
-    Platform: macos-1014
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_MacOS
-  condition: succeeded()
-  dependsOn: Build_Python_MacOS
-  variables:
-    VmImage: 'macOS-10.14'
-    Platform: macos-1014
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_Ubuntu_1604
-  dependsOn: []
-  variables:
-    VmImage: 'ubuntu-16.04'
-    Platform: ubuntu-1604
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_Ubuntu_1604
-  condition: succeeded()
-  dependsOn: Build_Python_Ubuntu_1604
-  variables:
-    VmImage: 'ubuntu-16.04'
-    Platform: ubuntu-1604
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_Ubuntu_1804
-  dependsOn: []
-  variables:
-    VmImage: 'ubuntu-18.04'
-    Platform: ubuntu-1804
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_Ubuntu_1804
-  condition: succeeded()
-  dependsOn: Build_Python_Ubuntu_1804
-  variables:
-    VmImage: 'ubuntu-18.04'
-    Platform: ubuntu-1804
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_Ubuntu_2004
-  dependsOn: []
-  variables:
-    VmImage: 'ubuntu-20.04'
-    Platform: ubuntu-2004
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_Ubuntu_2004
-  condition: succeeded()
-  dependsOn: Build_Python_Ubuntu_2004
-  variables:
-    VmImage: 'ubuntu-20.04'
-    Platform: ubuntu-2004
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_X64_Windows
-  dependsOn: []
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: windows-2016
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_x64_Windows
-  condition: succeeded()
-  dependsOn: Build_Python_X64_Windows
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: windows-2016
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_x86_Windows
-  dependsOn: []
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: windows-2016
-    Architecture: x86
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_x86_Windows
-  condition: succeeded()
-  dependsOn: Build_Python_x86_Windows
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: windows-2016
-    Architecture: x86
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml

azure-pipelines/run-ci-builds.yml

@@ -1,34 +0,0 @@
-trigger: none
-pr:
-  autoCancel: true
-  branches:
-    include:
-    - master
-  paths:
-    exclude:
-    - versions-manifest.json
-
-jobs:
-- job: Run_Builds
-  pool:
-    name: Azure Pipelines
-    vmImage: 'ubuntu-latest'
-  timeoutInMinutes: 180
-
-  steps:
-  - checkout: self
-    submodules: true
-
-  - task: PowerShell@2
-    displayName: 'Run build'
-    inputs:
-      targetType: filePath
-      filePath: './helpers/azure-devops/run-ci-builds.ps1 '
-      arguments: |
-          -TeamFoundationCollectionUri $(System.TeamFoundationCollectionUri) `
-          -AzureDevOpsProjectName $(System.TeamProject) `
-          -AzureDevOpsAccessToken $(System.AccessToken) `
-          -SourceBranch $(Build.SourceBranch) `
-          -SourceVersion $(Build.SourceVersion) `
-          -ToolVersions "$(PYTHON_VERSIONS)" `
-          -DefinitionId $(DEFINITION_ID) 

azure-pipelines/templates/build-job.yml

@@ -1,22 +0,0 @@
-jobs:
-- job: Build_Python
-  timeoutInMinutes: 90
-  pool:
-    name: Azure Pipelines
-    vmImage: $(VmImage)
-  steps:
-  - checkout: self
-    submodules: true
-    
-  - task: PowerShell@2
-    displayName: 'Build Python $(VERSION)'
-    inputs:
-      targetType: filePath
-      filePath: './builders/build-python.ps1'
-      arguments: '-Version $(VERSION) -Platform $(Platform) -Architecture $(Architecture)'
-
-  - task: PublishPipelineArtifact@1
-    displayName: 'Publish Artifact: Python $(VERSION)'
-    inputs:
-      targetPath: '$(Build.ArtifactStagingDirectory)' 
-      artifactName: 'python-$(VERSION)-$(Platform)-$(Architecture)'

azure-pipelines/templates/test-job.yml

@@ -1,80 +0,0 @@
-jobs:
-- job: Test_Python
-  pool: 
-    name: Azure Pipelines
-    vmImage: $(VmImage)
-  steps:
-  - checkout: self
-    submodules: true
-
-  - task: DownloadPipelineArtifact@2
-    inputs:
-      source: 'current'
-      artifact: 'python-$(VERSION)-$(Platform)-$(Architecture)'
-      path: $(Build.BinariesDirectory)
-
-  - task: ExtractFiles@1
-    inputs:
-      archiveFilePatterns: '$(Build.BinariesDirectory)/python-$(VERSION)-$(Platform)-$(Architecture).*'
-      destinationFolder: $(Build.BinariesDirectory)
-      cleanDestinationFolder: false
-
-  - task: PowerShell@2
-    displayName: 'Apply build artifact to the local machines'
-    inputs:
-      TargetType: inline
-      script: |
-        if ($env:PLATFORM -match 'windows') { powershell ./setup.ps1 } else { sh ./setup.sh }
-      workingDirectory: '$(Build.BinariesDirectory)'
-
-  - task: UsePythonVersion@0
-    displayName: 'Use Python $(VERSION)'
-    inputs:
-      versionSpec: '$(VERSION)'
-      architecture: '$(Architecture)'
-
-  - task: PowerShell@2
-    displayName: 'Verbose sysconfig dump'
-    inputs:
-      TargetType: inline
-      script: |
-        Invoke-Expression "python ./sources/python-config-output.py"
-      workingDirectory: '$(Build.SourcesDirectory)/tests'
-    condition: ne(variables['Platform'], 'windows-2016')
-
-  - task: PowerShell@2
-    displayName: 'Verbose python binary links'
-    inputs:
-      TargetType: inline
-      script: |
-        $pythonLocation = which python
-        if ($env:PLATFORM -match 'macos') { otool -L $pythonLocation } else { ldd $pythonLocation }
-      workingDirectory: '$(Build.BinariesDirectory)'
-    condition: ne(variables['Platform'], 'windows-2016')
-
-  - task: PowerShell@2
-    displayName: 'Run tests'
-    inputs:
-      TargetType: inline
-      script: |
-        Install-Module Pester -Force -Scope CurrentUser -RequiredVersion 4.10.1
-        Import-Module Pester
-        $pesterParams = @{
-          Path="./python-tests.ps1";
-          Parameters=@{
-            Version="$(VERSION)";
-            Platform="$(Platform)";
-          }
-        }
-        Invoke-Pester -Script $pesterParams -OutputFile "test_results.xml" -OutputFormat NUnitXml
-      workingDirectory: '$(Build.SourcesDirectory)/tests'
-
-  - task: PublishTestResults@2
-    displayName: 'Publish test results'
-    inputs:
-      testResultsFiles: '*.xml'
-      testResultsFormat: NUnit
-      searchFolder: 'tests'
-      failTaskOnFailedTests: true
-      testRunTitle: "$(TestRunTitle)"
-    condition: always()

builders/build-python.ps1

@@ -1,6 +1,6 @@
-using module "./builders/win-python-builder.psm1"
-using module "./builders/ubuntu-python-builder.psm1"
-using module "./builders/macos-python-builder.psm1"
+using module "./win-python-builder.psm1"
+using module "./ubuntu-python-builder.psm1"
+using module "./macos-python-builder.psm1"
 
 <#
 .SYNOPSIS
@@ -21,7 +21,7 @@ Required parameter. The platform for which Python will be built.
 #>
 
 param(
-    [Parameter (Mandatory=$true)][Version] $Version,
+    [Parameter (Mandatory=$true)][semver] $Version,
     [Parameter (Mandatory=$true)][string] $Platform,
     [string] $Architecture = "x64"
 )
@@ -29,6 +29,7 @@ param(
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "common-helpers.psm1") -DisableNameChecking
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "nix-helpers.psm1") -DisableNameChecking
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "win-helpers.psm1") -DisableNameChecking
+Import-Module (Join-Path $PSScriptRoot "python-version.psm1") -DisableNameChecking
 
 function Get-PythonBuilder {
     <#
@@ -49,18 +50,17 @@ function Get-PythonBuilder {
 
     #>
 
-    param (
-        [version] $Version,
+    param(
+        [semver] $Version,
         [string] $Architecture,
         [string] $Platform
     )
 
-    $Platform = $Platform.ToLower()  
-    if ($Platform -match 'windows') {
+    if ($Platform -match 'win32') {
         $builder = [WinPythonBuilder]::New($Version, $Architecture, $Platform)
-    } elseif ($Platform -match 'ubuntu') {
+    } elseif ($Platform -match 'linux') {
         $builder = [UbuntuPythonBuilder]::New($Version, $Architecture, $Platform)
-    } elseif ($Platform -match 'macos') {
+    } elseif ($Platform -match 'darwin') {
         $builder = [macOSPythonBuilder]::New($Version, $Architecture, $Platform)
     } else {
         Write-Host "##vso[task.logissue type=error;] Invalid platform: $Platform"
@@ -71,5 +71,5 @@ function Get-PythonBuilder {
 }
 
 ### Create Python builder instance, and build artifact
-$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture  -Platform $Platform 
+$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture -Platform $Platform 
 $Builder.Build()

builders/invoke-workflow.psm1

@@ -0,0 +1,52 @@
+function Invoke-Workflow {
+    param (
+        [string] $Version,
+        [string] $PublishRelease
+    )
+    
+    $payload = @{
+        "ref" = "main"
+        "inputs" = @{
+            "VERSION" = "$Version"
+            "PUBLISH_RELEASES" = "$PublishRelease"
+        }
+    } | ConvertTo-Json
+    $headers = @{
+        Authorization="Bearer $env:TOKEN"
+    }
+    $actionsRepoUri = "$env:GITHUB_API_URL/repos/$env:GITHUB_REPOSITORY/actions"
+    Invoke-RestMethod -uri "$actionsRepoUri/workflows/build-python-packages.yml/dispatches" -method POST -headers $headers -body $payload
+
+    $result = [PSCustomObject]@{
+        Version = $Version
+        Conclusion = "failure"
+        Url = "Not run"
+    }
+    # Triggering workflow and verifying that it has been triggered with retries
+    while (-not $workflowToCheck) {
+        Start-Sleep -seconds 40
+        $workflowRuns = (Invoke-RestMethod "$actionsRepoUri/runs").workflow_runs | Where-Object {$_.status -like "*progress*" -and $_.id -ne $env:GITHUB_RUN_ID}
+        $workflowToCheck = $workflowRuns | Where-Object {
+            (Invoke-RestMethod "$actionsRepoUri/runs/$($_.id)/jobs").jobs.steps.name -like "*$Version"
+        }
+        $retries++
+        if ($retries -gt 10) {
+            Write-Host "Workflow triggered for version '$Version' not found or something went wrong with fetching the workflow status"
+            return $result
+        }
+    }
+    # Waiting for workflow to complete
+    while ($workflowToCheck.status -ne "completed") {
+        Start-Sleep -Seconds 120
+        $workflowToCheck = Invoke-RestMethod "$actionsRepoUri/runs/$($workflowToCheck.id)"
+        Write-Host "Workflow run with Id: $($workflowToCheck.id) for version '$Version' - status '$($workflowToCheck.status)'"
+    }
+    $result.Conclusion = $workflowToCheck.conclusion
+    $result.Url = $workflowToCheck.html_url
+    if ($workflowToCheck.conclusion -ne "success") {
+        Write-Host "Triggered workflow for version '$Version' completed unsuccessfully with result '$($workflowToCheck.conclusion)'. Check the logs: $($workflowToCheck.html_url)"
+        return $result
+    }
+    Write-Host "Triggered workflow for version '$Version' succeeded; Url: $($workflowToCheck.html_url)"
+    return $result
+}

builders/macos-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/nix-python-builder.psm1"
+using module "./nix-python-builder.psm1"
 
 class macOSPythonBuilder : NixPythonBuilder {
     <#
@@ -8,6 +8,10 @@ class macOSPythonBuilder : NixPythonBuilder {
     .DESCRIPTION
     Contains methods that required to build macOS Python artifact from sources. Inherited from base NixPythonBuilder.
 
+    While python.org provides precompiled binaries for macOS, switching to them risks breaking existing customers.
+    If we wanted to start using the official binaries instead of building from source, we should avoid changing previous versions
+    so we remain backwards compatible.
+
     .PARAMETER platform
     The full name of platform for which Python should be built.
 
@@ -17,11 +21,18 @@ class macOSPythonBuilder : NixPythonBuilder {
     #>
 
     macOSPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) { }
 
+    [void] PrepareEnvironment() {
+        <#
+        .SYNOPSIS
+        Prepare system environment by installing dependencies and required packages.
+        #>
+    }
+
     [void] Configure() {
         <#
         .SYNOPSIS
@@ -29,30 +40,156 @@ class macOSPythonBuilder : NixPythonBuilder {
         #>
 
         $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation()
-        $configureString = "./configure --prefix=$pythonBinariesLocation --enable-optimizations --enable-shared --with-lto"
+        $configureString = "./configure"
+        $configureString += " --prefix=$pythonBinariesLocation"
+        $configureString += " --enable-optimizations"
+        $configureString += " --enable-shared"
+        $configureString += " --with-lto"
+
+        ### For Python versions which support it, compile a universal2 (arm64 + x86_64 hybrid) build. The arm64 slice
+        ### will never be used itself by a Github Actions runner but using a universal2 Python is the only way to build
+        ### universal2 C extensions and wheels. This is supported by Python >= 3.10 and was backported to Python >=
+        ### 3.9.1 and >= 3.8.10.
 
         ### OS X 10.11, Apple no longer provides header files for the deprecated system version of OpenSSL.
         ### Solution is to install these libraries from a third-party package manager,
         ### and then add the appropriate paths for the header and library files to configure command.
         ### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies)
-        if ($this.Version -lt "3.7.0") {
-            $env:LDFLAGS="-L$(brew --prefix openssl)/lib"
-            $env:CFLAGS="-I$(brew --prefix openssl)/include"
-        } else {
-            $configureString += " --with-openssl=/usr/local/opt/openssl"
-        }
+        $configureString += " --with-openssl=/usr/local/opt/openssl@3"
+        $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include/tcl-tk' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'"
+
+        ### Compile with support of loadable sqlite extensions.
+        ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
+        $configureString += " --enable-loadable-sqlite-extensions"
+        $env:LDFLAGS += " -L$(brew --prefix sqlite3)/lib"
+        $env:CFLAGS += " -I$(brew --prefix sqlite3)/include"
+        $env:CPPFLAGS += "-I$(brew --prefix sqlite3)/include"
+
+        Write-Host "The passed configure options are: "
+        Write-Host $configureString
 
         Execute-Command -Command $configureString
     }
 
-    [void] PrepareEnvironment() {
+    [string] GetPkgName() {
         <#
         .SYNOPSIS
-        Prepare system environment by installing dependencies and required packages.
+        Return Python installation Package.
         #>
 
-        ### reinstall header files to Avoid issue with X11 headers on Mojave
-        $pkgName = "/Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg"
-        Execute-Command -Command "sudo installer -pkg $pkgName -target /"
+        $nativeVersion = Convert-Version -version $this.Version
+        $architecture = "-macos11"
+        $extension = ".pkg"
+
+        $pkg = "python-${nativeVersion}${architecture}${extension}"
+
+        return $pkg
+    }
+
+    [uri] GetPkgUri() {
+        <#
+        .SYNOPSIS
+        Get base Python URI and return complete URI for Python installation package.
+        #>
+
+        $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $pkg = $this.GetPkgName()
+
+        $uri = "${base}/${versionName}/${pkg}"
+
+        return $uri
+    }
+
+    [string] DownloadPkg() {
+        <#
+        .SYNOPSIS
+        Download Python installation executable into artifact location.
+        #>
+
+        $pkgUri = $this.GetPkgUri()
+
+        Write-Host "Sources URI: $pkgUri"
+        $pkgLocation = Download-File -Uri $pkgUri -OutputFolder $this.WorkFolderLocation
+        Write-Debug "Done; Package location: $pkgLocation"
+
+        New-Item -Path $this.WorkFolderLocation -Name "build_output.txt"  -ItemType File
+        return $pkgLocation
+    }
+
+    [string] GetFrameworkName() {
+        <#
+        .SYNOPSIS
+        Get the Python installation Package name.
+        #>
+
+        if ($this.IsFreeThreaded()) {
+            return "PythonT.framework"
+        } else {
+            return "Python.framework"
+        }
+    }
+
+    [string] GetPkgChoices() {
+        <#
+        .SYNOPSIS
+        Reads the configuration XML file for the Python installer
+        #>
+
+        $config = if ($this.IsFreeThreaded()) { "freethreaded" } else { "default" }
+        $choicesFile = Join-Path $PSScriptRoot "../config/macos-pkg-choices-$($config).xml"
+        $choicesTemplate = Get-Content -Path $choicesFile -Raw
+
+        $variablesToReplace = @{
+            "{{__VERSION_MAJOR_MINOR__}}" = "$($this.Version.Major).$($this.Version.Minor)";
+        }
+
+        $variablesToReplace.keys | ForEach-Object { $choicesTemplate = $choicesTemplate.Replace($_, $variablesToReplace[$_]) }
+        return $choicesTemplate
+    }
+
+    [void] CreateInstallationScriptPkg() {
+        <#
+        .SYNOPSIS
+        Create Python artifact installation script based on specified template.
+        #>
+
+        $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath "macos-pkg-setup-template.sh"
+        $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
+        $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName  -ItemType File
+
+        $variablesToReplace = @{
+            "{{__VERSION_FULL__}}" = $this.Version;
+            "{{__PKG_NAME__}}" = $this.GetPkgName();
+            "{{__ARCH__}}" = $this.Architecture;
+            "{{__FRAMEWORK_NAME__}}" = $this.GetFrameworkName();
+            "{{__PKG_CHOICES__}}" = $this.GetPkgChoices();
+        }
+
+        $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
+        $installationTemplateContent | Out-File -FilePath $installationScriptLocation
+        Write-Debug "Done; Installation script location: $installationScriptLocation)"
+    }
+
+    [void] Build() {
+        <#
+        .SYNOPSIS
+        Generates Python artifact from downloaded Python installation executable.
+        #>
+
+        $PkgVersion = [semver]"3.11.0-beta.1"
+
+        if (($this.Version -ge $PkgVersion) -or ($this.Architecture -eq "arm64")) {
+            Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..."
+            $this.DownloadPkg()
+
+            Write-Host "Create installation script..."
+            $this.CreateInstallationScriptPkg()
+        } else {
+            ([NixPythonBuilder]$this).Build()
+        }
+
+        Write-Host "Archive artifact"
+        $this.ArchiveArtifact()
     }
 }

builders/nix-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/python-builder.psm1"
+using module "./python-builder.psm1"
 
 class NixPythonBuilder : PythonBuilder {
     <#
@@ -33,13 +33,12 @@ class NixPythonBuilder : PythonBuilder {
     [string] $OutputArtifactName
 
     NixPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) {
         $this.InstallationTemplateName = "nix-setup-template.sh"	
         $this.InstallationScriptName = "setup.sh"
-
         $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.tar.gz"
     }
 
@@ -50,8 +49,10 @@ class NixPythonBuilder : PythonBuilder {
         #>
 
         $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $nativeVersion = Convert-Version -version $this.Version
 
-        return "${base}/$($this.Version)/Python-$($this.Version).tgz"
+        return "${base}/${versionName}/Python-${nativeVersion}.tgz"
     }
 
     [string] GetPythonBinary() {
@@ -60,8 +61,7 @@ class NixPythonBuilder : PythonBuilder {
         Return name of Python binary.
         #>
 
-        if ($this.Version.Major -eq 2) { $pythonBinary = "python" } else { $pythonBinary = "python3" }
-        return $pythonBinary
+        return "python3"
     }
 
     [string] Download() {
@@ -95,9 +95,8 @@ class NixPythonBuilder : PythonBuilder {
         $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
 
         $variablesToReplace = @{
-            "{{__VERSION_MAJOR__}}" = $this.Version.Major;
-            "{{__VERSION_MINOR__}}" = $this.Version.Minor;
-            "{{__VERSION_BUILD__}}" = $this.Version.Build;
+            "{{__VERSION_FULL__}}" = $this.Version;
+            "{{__ARCH__}}" = $this.Architecture;
         }
         $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
 
@@ -115,7 +114,7 @@ class NixPythonBuilder : PythonBuilder {
         Write-Debug "make Python $($this.Version)-$($this.Architecture) $($this.Platform)"
         $buildOutputLocation = New-Item -Path $this.WorkFolderLocation -Name "build_output.txt" -ItemType File
         
-        Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue	
+        Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue
         Execute-Command -Command "make install" -ErrorAction Continue
 
         Write-Debug "Done; Make log location: $buildOutputLocation"

builders/python-builder.psm1

@@ -29,7 +29,7 @@ class PythonBuilder {
 
     #>
 
-    [version] $Version
+    [semver] $Version
     [string] $Architecture
     [string] $Platform
     [string] $HostedToolcacheLocation
@@ -38,17 +38,23 @@ class PythonBuilder {
     [string] $ArtifactFolderLocation
     [string] $InstallationTemplatesLocation
 
-    PythonBuilder ([version] $version, [string] $architecture, [string] $platform) {
+    PythonBuilder ([semver] $version, [string] $architecture, [string] $platform) {
+        $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers"
+
+        $artifactDirectory = Join-Path $env:RUNNER_TEMP "artifact"
+        $workDirectory = Join-Path $env:RUNNER_TEMP "work"
+
+        New-Item -Force -Type Directory $artifactDirectory
+        New-Item -Force -Type Directory $workDirectory
+
+        $this.HostedToolcacheLocation = $env:RUNNER_TOOL_CACHE
+        $this.TempFolderLocation = $env:RUNNER_TEMP
+        $this.WorkFolderLocation =  $workDirectory
+        $this.ArtifactFolderLocation = $artifactDirectory
+
         $this.Version = $version
         $this.Architecture = $architecture
         $this.Platform = $platform
-
-        $this.HostedToolcacheLocation = $env:AGENT_TOOLSDIRECTORY
-        $this.TempFolderLocation = $env:BUILD_SOURCESDIRECTORY
-        $this.WorkFolderLocation = $env:BUILD_BINARIESDIRECTORY
-        $this.ArtifactFolderLocation = $env:BUILD_STAGINGDIRECTORY
-
-        $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers"
     }
 
     [uri] GetBaseUri() {
@@ -79,11 +85,39 @@ class PythonBuilder {
         return "$pythonToolcacheLocation/$($this.Version)/$($this.Architecture)"
     }
 
+    [string] GetBaseVersion() {
+        <#
+        .SYNOPSIS
+        Return Major.Minor.Patch version string.
+        #>
+
+        return "$($this.Version.Major).$($this.Version.Minor).$($this.Version.Patch)"
+    }
+
+    [string] GetHardwareArchitecture() {
+        <#
+        .SYNOPSIS
+        The hardware architecture (x64, arm64) without any Python free threading suffix.
+        #>
+
+        return $this.Architecture.Replace("-freethreaded", "")
+    }
+
+    [bool] IsFreeThreaded() {
+        <#
+        .SYNOPSIS
+        Check if Python version is free threaded.
+        #>
+
+        return $this.Architecture.EndsWith("-freethreaded")
+    }
+
     [void] PreparePythonToolcacheLocation() {
         <#
         .SYNOPSIS
         Prepare system hostedtoolcache folder for new Python version. 
         #>
+        
         $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation()
 
         if (Test-Path $pythonBinariesLocation) {

builders/python-version.psm1

@@ -0,0 +1,45 @@
+function Convert-Label() {
+    <#
+    .SYNOPSIS
+    Convert generic semver label to native Python label.
+    #>
+
+    param(
+        [Parameter(Mandatory)]
+        [string] $label
+    )
+
+    switch ($label) {
+        "alpha" { return "a" }
+        "beta" { return "b" }
+        "rc" { return "rc" }
+        default { throw "Invalid version label '$label'" }
+    }
+}
+
+function Convert-Version {
+    <#
+    .SYNOPSIS
+    Convert generic semver version to native Python version.
+    #>
+
+    param(
+        [Parameter(Mandatory)]
+        [semver] $version,
+        [char] $delimiter = "."
+    )
+
+    $nativeVersion = "{0}.{1}.{2}" -f $version.Major, $version.Minor, $version.Patch
+
+    if ($version.PreReleaseLabel)
+    {
+        $preReleaseLabel = $version.PreReleaseLabel.Split($delimiter)
+        
+        $preReleaseLabelName = Convert-Label -Label $preReleaseLabel[0]
+        $preReleaseLabelVersion = $preReleaseLabel[1]
+
+        $nativeVersion += "${preReleaseLabelName}${preReleaseLabelVersion}"
+    }
+
+    return $nativeVersion
+}

builders/python-versions-runner.ps1

@@ -0,0 +1,31 @@
+<#
+.SYNOPSIS
+Generate Python artifact.
+
+.DESCRIPTION
+Script that triggering and fetching the result of the "Build python package" workflows with provided python versions
+
+.PARAMETER Version
+Required parameter. Python versions to trigger builds for.
+
+.PARAMETER PublishRelease
+Switch parameter. Whether to publish release for built version.
+
+#>
+
+param(
+    [Parameter (Mandatory=$true, HelpMessage="Python version to trigger build for")]
+    [array] $Versions,
+    [Parameter (Mandatory=$false, HelpMessage="Whether to publish release for built version")]
+    [string] $PublishRelease
+)
+
+$summary = $Versions | ForEach-Object -Parallel { 
+    Import-Module "./builders/invoke-workflow.psm1"
+    Invoke-Workflow -Version $_ -PublishRelease $Using:PublishRelease
+}
+Write-Host "Results of triggered workflows:"
+$summary | Out-String
+if ($summary.Conclusion -contains "failure" -or $summary.Conclusion -contains "cancelled") {
+    exit 1
+}

builders/ubuntu-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/nix-python-builder.psm1"
+using module "./nix-python-builder.psm1"
 
 class UbuntuPythonBuilder : NixPythonBuilder {
     <#
@@ -17,7 +17,7 @@ class UbuntuPythonBuilder : NixPythonBuilder {
     #>
 
     UbuntuPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) { }
@@ -32,13 +32,26 @@ class UbuntuPythonBuilder : NixPythonBuilder {
 
         ### To build Python with SO we must pass full path to lib folder to the linker
         $env:LDFLAGS="-Wl,--rpath=${pythonBinariesLocation}/lib"
-        $configureString = "./configure --prefix=$pythonBinariesLocation --enable-shared --enable-optimizations"
+        $configureString = "./configure"
+        $configureString += " --prefix=$pythonBinariesLocation"
+        $configureString += " --enable-shared"
+        $configureString += " --enable-optimizations"
 
-        if ($this.Version -lt "3.0.0") {
-            ### Compile with ucs4 for Python 2.x. On 3.x, ucs4 is enabled by default
-            $configureString += " --enable-unicode=ucs4"
+        if ($this.IsFreeThreaded()) {
+            if ($this.Version -lt "3.13.0") {
+                Write-Host "Python versions lower than 3.13.0 do not support free threading"
+                exit 1
+            }
+            $configureString += " --disable-gil"
         }
 
+        ### Compile with support of loadable sqlite extensions.
+        ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
+        $configureString += " --enable-loadable-sqlite-extensions"
+
+        Write-Host "The passed configure options are: "
+        Write-Host $configureString
+
         Execute-Command -Command $configureString
     }
 
@@ -48,17 +61,15 @@ class UbuntuPythonBuilder : NixPythonBuilder {
         Prepare system environment by installing dependencies and required packages.
         #>
 
-        if (($this.Version -gt "3.0.0") -and ($this.Version -lt "3.5.3")) {
-            Write-Host "Python3 versions lower than 3.5.3 are not supported"
+        if ($this.Version -lt "3.9.0") {
+            Write-Host "Python versions lower than 3.9.0 are not supported"
             exit 1
         }
 
         ### Compile with tkinter support
-        if ($this.Version -gt "3.0.0") {
-            $tkinterInstallString = "sudo apt-get install -y --allow-downgrades python3-tk tk-dev"
-        } else {
-            $tkinterInstallString = "sudo apt install -y python-tk tk-dev"
-        }
+        $tkinterInstallString = "sudo apt-get install -y --allow-downgrades python3-tk tk-dev"
+
+        Execute-Command -Command "sudo apt-get update"
         Execute-Command -Command $tkinterInstallString
 
         ### Install dependent packages
@@ -71,15 +82,13 @@ class UbuntuPythonBuilder : NixPythonBuilder {
             "libsqlite3-dev",
             "libncursesw5-dev",
             "libreadline-dev",
-            "libgdbm-dev"
+            "libgdbm-dev",
+            "liblzma-dev"
         ) | ForEach-Object {
             Execute-Command -Command "sudo apt install -y $_"
         }
 
-        if ($this.Platform -ne "ubuntu-1604") {
-            ### On Ubuntu-1804, libgdbm-compat-dev has older modules that are no longer in libgdbm-dev
-            Execute-Command -Command "sudo apt install -y libgdbm-compat-dev"
-        }
-        
+        ### On Ubuntu-1804, libgdbm-compat-dev has older modules that are no longer in libgdbm-dev
+        Execute-Command -Command "sudo apt install -y libgdbm-compat-dev"
     }
 }

builders/win-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/python-builder.psm1"
+using module "./python-builder.psm1"
 
 class WinPythonBuilder : PythonBuilder {
     <#
@@ -27,7 +27,7 @@ class WinPythonBuilder : PythonBuilder {
     [string] $OutputArtifactName
 
     WinPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) {
@@ -42,9 +42,7 @@ class WinPythonBuilder : PythonBuilder {
         Return extension for required version of Python executable. 
         #>
 
-        $extension = if ($this.Version -lt "3.5" -and $this.Version -ge "2.5") { ".msi" } else { ".exe" }
-
-        return $extension
+        return ".exe"
     }
 
     [string] GetArchitectureExtension() {
@@ -54,12 +52,10 @@ class WinPythonBuilder : PythonBuilder {
         #>
 
         $ArchitectureExtension = ""
-        if ($this.Architecture -eq "x64") {
-            if ($this.Version -ge "3.5") {
-                $ArchitectureExtension = "-amd64"
-            } else {
-                $ArchitectureExtension = ".amd64"
-            }
+        if ($this.GetHardwareArchitecture() -eq "x64") {
+            $ArchitectureExtension = "-amd64"
+        } elseif ($this.GetHardwareArchitecture() -eq "arm64") {
+                $ArchitectureExtension = "-arm64"
         }
 
         return $ArchitectureExtension
@@ -72,10 +68,12 @@ class WinPythonBuilder : PythonBuilder {
         #>
 
         $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $nativeVersion = Convert-Version -version $this.Version
         $architecture = $this.GetArchitectureExtension()
         $extension = $this.GetPythonExtension()
 
-        $uri = "${base}/$($this.Version)/python-$($this.Version)${architecture}${extension}"
+        $uri = "${base}/${versionName}/python-${nativeVersion}${architecture}${extension}"
 
         return $uri
     }
@@ -109,6 +107,7 @@ class WinPythonBuilder : PythonBuilder {
 
         $variablesToReplace = @{
             "{{__ARCHITECTURE__}}" = $this.Architecture;
+            "{{__HARDWARE_ARCHITECTURE__}}" = $this.GetHardwareArchitecture();
             "{{__VERSION__}}" = $this.Version;
             "{{__PYTHON_EXEC_NAME__}}" = $pythonExecName
         }

config/macos-pkg-choices-default.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<array>
+    <dict>
+   </dict>
+</array>
+</plist>

config/macos-pkg-choices-freethreaded.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<array>
+    <dict>
+        <key>attributeSetting</key>
+        <integer>1</integer>
+        <key>choiceAttribute</key>
+        <string>selected</string>
+        <key>choiceIdentifier</key>
+        <string>org.python.Python.PythonTFramework-{{__VERSION_MAJOR_MINOR__}}</string>
+    </dict>
+</array>
+</plist>

config/python-manifest-config.json

@@ -1,8 +1,8 @@
 {
-    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+)-([\\w\\.]+)?-?(x\\d+)",
+    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+(-freethreaded)?)",
     "groups": {
-        "arch": 3,
-        "platform": 1,
-        "platform_version": 2
+        "arch": 4,
+        "platform": 2,
+        "platform_version": 3
     }
 }

installers/macos-pkg-setup-template.sh

@@ -0,0 +1,94 @@
+set -e
+
+PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
+PYTHON_PKG_NAME="{{__PKG_NAME__}}"
+PYTHON_FRAMEWORK_NAME="{{__FRAMEWORK_NAME__}}"
+PYTHON_PKG_CHOICES=$(cat << 'EOF'
+{{__PKG_CHOICES__}}
+EOF
+)
+ARCH="{{__ARCH__}}"
+MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
+MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
+
+PYTHON_MAJOR=python$MAJOR_VERSION
+PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION
+PYTHON_MAJOR_MINOR=python$MAJOR_VERSION$MINOR_VERSION
+
+if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then
+    # No AGENT_TOOLSDIRECTORY on GitHub images
+    TOOLCACHE_ROOT=$RUNNER_TOOL_CACHE
+else
+    TOOLCACHE_ROOT=$AGENT_TOOLSDIRECTORY
+fi
+
+PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
+PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
+PYTHON_FRAMEWORK_PATH="/Library/Frameworks/${PYTHON_FRAMEWORK_NAME}/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
+PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
+
+echo "Check if Python hostedtoolcache folder exist..."
+if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
+    echo "Creating Python hostedtoolcache folder..."
+    mkdir -p $PYTHON_TOOLCACHE_PATH
+else
+    # remove ALL other directories for same major.minor python versions
+    find $PYTHON_TOOLCACHE_PATH -name "${MAJOR_VERSION}.${MINOR_VERSION}.*"|while read python_version;do
+        python_version_arch="$python_version/$ARCH"
+        if [ -e "$python_version_arch" ];then
+            echo "Deleting Python $python_version_arch"
+            rm -rf "$python_version_arch"
+        fi
+    done
+fi
+
+PYTHON_PKG_CHOICES_FILES=$(mktemp)
+echo "$PYTHON_PKG_CHOICES" > $PYTHON_PKG_CHOICES_FILES
+
+echo "Install Python binaries from prebuilt package"
+sudo installer -pkg $PYTHON_PKG_NAME -applyChoiceChangesXML $PYTHON_PKG_CHOICES_FILES -target /
+
+echo "Create hostedtoolcach symlinks (Required for the backward compatibility)"
+echo "Create Python $PYTHON_FULL_VERSION folder"
+mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+
+ln -s "${PYTHON_FRAMEWORK_PATH}/bin" bin
+ln -s "${PYTHON_FRAMEWORK_PATH}/include" include
+ln -s "${PYTHON_FRAMEWORK_PATH}/share" share
+ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib
+
+echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
+ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
+chmod +x python
+
+# Note that bin is a symlink so referencing .. from bin will not work as expected
+cd bin/
+
+# This symlink already exists if Python version with the same major.minor version is installed,
+# since we do not remove the framework folder
+if [ ! -f $PYTHON_MAJOR_MINOR ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
+fi
+
+if [ ! -f $PYTHON_MAJOR ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR
+fi
+
+if [ ! -f python ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR python
+fi
+
+chmod +x $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
+
+echo "Upgrading pip..."
+export PIP_ROOT_USER_ACTION=ignore
+./python -m ensurepip
+./python -m pip install --upgrade --force-reinstall pip --disable-pip-version-check --no-warn-script-location
+
+echo "Install OpenSSL certificates"
+sh -e "${PYTHON_APPLICATION_PATH}/Install Certificates.command"
+
+echo "Create complete file"
+touch $PYTHON_TOOLCACHE_VERSION_PATH/${ARCH}.complete

installers/map_python_platforms.json

@@ -1,29 +0,0 @@
-{
-    "macos-1014": [
-        {
-            "platform": "darwin",
-            "platform_version": "10.14"
-        },
-        {
-            "platform": "darwin",
-            "platform_version": "10.15"
-        }
-    ],
-    "ubuntu-1604": [
-        {
-            "platform": "linux",
-            "platform_version": "16.04"
-        }
-    ],
-    "ubuntu-1804": [
-        {
-            "platform": "linux",
-            "platform_version": "18.04"
-        }
-    ],
-    "windows-2016": [
-        {
-            "platform": "win32"
-        }
-    ]
-}

installers/nix-setup-template.sh

@@ -1,13 +1,13 @@
 set -e
 
-MAJOR_VERSION="{{__VERSION_MAJOR__}}"
-MINOR_VERSION="{{__VERSION_MINOR__}}"
-BUILD_VERSION="{{__VERSION_BUILD__}}"
+PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
+ARCH="{{__ARCH__}}"
+MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
+MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
 
 PYTHON_MAJOR=python$MAJOR_VERSION
 PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION
 PYTHON_MAJORMINOR=python$MAJOR_VERSION$MINOR_VERSION
-PYTHON_FULL_VERSION=$MAJOR_VERSION.$MINOR_VERSION.$BUILD_VERSION
 
 if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then
     # No AGENT_TOOLSDIRECTORY on GitHub images
@@ -18,15 +18,15 @@ fi
 
 PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
 PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
-PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
 
 echo "Check if Python hostedtoolcache folder exist..."
 if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
     echo "Creating Python hostedtoolcache folder..."
     mkdir -p $PYTHON_TOOLCACHE_PATH
-elif [ -d $PYTHON_TOOLCACHE_VERSION_PATH ]; then
-    echo "Deleting Python $PYTHON_FULL_VERSION"
-    rm -rf $PYTHON_TOOLCACHE_VERSION_PATH
+elif [ -d $PYTHON_TOOLCACHE_VERSION_ARCH_PATH ]; then
+    echo "Deleting Python $PYTHON_FULL_VERSION ($ARCH)"
+    rm -rf $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
 fi
 
 echo "Create Python $PYTHON_FULL_VERSION folder"
@@ -49,9 +49,10 @@ fi
 
 chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR python
 
-echo "Upgrading PIP..."
+echo "Upgrading pip..."
+export PIP_ROOT_USER_ACTION=ignore
 ./python -m ensurepip
-./python -m pip install --ignore-installed pip
+./python -m pip install --upgrade --force-reinstall pip --disable-pip-version-check --no-warn-script-location
 
 echo "Create complete file"
-touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete
+touch $PYTHON_TOOLCACHE_VERSION_PATH/$ARCH.complete

installers/win-setup-template.ps1

@@ -1,51 +1,43 @@
 [String] $Architecture = "{{__ARCHITECTURE__}}"
-[Version] $Version = "{{__VERSION__}}"
+[String] $HardwareArchitecture = "{{__HARDWARE_ARCHITECTURE__}}"
+[String] $Version = "{{__VERSION__}}"
 [String] $PythonExecName = "{{__PYTHON_EXEC_NAME__}}"
 
 function Get-RegistryVersionFilter {
-    param
-    (
+    param(
         [Parameter(Mandatory)][String] $Architecture,
         [Parameter(Mandatory)][Int32] $MajorVersion,
         [Parameter(Mandatory)][Int32] $MinorVersion
     )
 
     $archFilter = if ($Architecture -eq 'x86') { "32-bit" } else { "64-bit" }
-    ### Python 2.7 x86 have no architecture postfix
-    if (($Architecture -eq "x86") -and ($MajorVersion -eq 2))
-    {
-        "Python $MajorVersion.$MinorVersion.\d+$"
-    }
-    else
-    {
-        "Python $MajorVersion.$MinorVersion.*($archFilter)"
-    }
+    "Python $MajorVersion.$MinorVersion.*($archFilter)"
 }
 
-function Remove-RegistryEntries
-{
-    param
-    (
+function Remove-RegistryEntries {
+    param(
         [Parameter(Mandatory)][String] $Architecture,
         [Parameter(Mandatory)][Int32] $MajorVersion,
         [Parameter(Mandatory)][Int32] $MinorVersion
     )
 
-    $versionFilter = Get-RegistryVersionFilter -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
+    $versionFilter = Get-RegistryVersionFilter -Architecture $HardwareArchitecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
 
     $regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products"
-    $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName
-    foreach ($key in $regKeys)
-    {
-        if ($key.getValue("DisplayName") -match $versionFilter)
-        {
-            Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose
+    if (Test-Path -Path Registry::$regPath) {
+        $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName
+        foreach ($key in $regKeys) {
+            if ($key.getValue("DisplayName") -match $versionFilter) {
+                Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose
+            }
         }
     }
 
     $regPath = "HKEY_CLASSES_ROOT\Installer\Products"
-    Get-ChildItem -Path Registry::$regPath | Where-Object { $_.GetValue("ProductName") -match $versionFilter } | ForEach-Object {
-        Remove-Item Registry::$_ -Recurse -Force -Verbose
+    if (Test-Path -Path Registry::$regPath) {
+        Get-ChildItem -Path Registry::$regPath | Where-Object { $_.GetValue("ProductName") -match $versionFilter } | ForEach-Object {
+            Remove-Item Registry::$_ -Recurse -Force -Verbose
+        }
     }
 
     $uninstallRegistrySections = @(
@@ -63,40 +55,37 @@ function Remove-RegistryEntries
 }
 
 function Get-ExecParams {
-    param
-    (
+    param(
         [Parameter(Mandatory)][Boolean] $IsMSI,
+        [Parameter(Mandatory)][Boolean] $IsFreeThreaded,
         [Parameter(Mandatory)][String] $PythonArchPath
     )
 
-    if ($IsMSI) 
-    {
+    if ($IsMSI) {
         "TARGETDIR=$PythonArchPath ALLUSERS=1"
-    } 
-    else 
-    {
-        "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1"
+    } else {
+        $Include_freethreaded = if ($IsFreeThreaded) { "Include_freethreaded=1" } else { "" }
+        "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1 $Include_freethreaded"
     }
 }
 
 $ToolcacheRoot = $env:AGENT_TOOLSDIRECTORY
-if ([string]::IsNullOrEmpty($ToolcacheRoot))
-{
+if ([string]::IsNullOrEmpty($ToolcacheRoot)) {
     # GitHub images don't have `AGENT_TOOLSDIRECTORY` variable
     $ToolcacheRoot = $env:RUNNER_TOOL_CACHE
 }
 $PythonToolcachePath = Join-Path -Path $ToolcacheRoot -ChildPath "Python"
-$PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version.ToString()
+$PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version
 $PythonArchPath = Join-Path -Path $PythonVersionPath -ChildPath $Architecture
 
 $IsMSI = $PythonExecName -match "msi"
+$IsFreeThreaded = $Architecture -match "-freethreaded"
 
-$MajorVersion = $Version.Major
-$MinorVersion = $Version.Minor
+$MajorVersion = $Version.Split('.')[0]
+$MinorVersion = $Version.Split('.')[1]
 
 Write-Host "Check if Python hostedtoolcache folder exist..."
-if (-Not (Test-Path $PythonToolcachePath)) 
-{
+if (-Not (Test-Path $PythonToolcachePath)) {
     Write-Host "Create Python toolcache folder"
     New-Item -ItemType Directory -Path $PythonToolcachePath | Out-Null
 }
@@ -104,22 +93,19 @@ if (-Not (Test-Path $PythonToolcachePath))
 Write-Host "Check if current Python version is installed..."
 $InstalledVersions = Get-Item "$PythonToolcachePath\$MajorVersion.$MinorVersion.*\$Architecture"
 
-if ($null -ne $InstalledVersions)
-{
+if ($null -ne $InstalledVersions) {
     Write-Host "Python$MajorVersion.$MinorVersion ($Architecture) was found in $PythonToolcachePath..."
 
-    foreach ($InstalledVersion in $InstalledVersions)
-    {
-        if (Test-Path -Path $InstalledVersion)
-        {
+    foreach ($InstalledVersion in $InstalledVersions) {
+        if (Test-Path -Path $InstalledVersion) {
             Write-Host "Deleting $InstalledVersion..."
             Remove-Item -Path $InstalledVersion -Recurse -Force
-            Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose
+            if (Test-Path -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete") {
+                Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose
+            }
         }
     }
-} 
-else 
-{
+} else {
     Write-Host "No Python$MajorVersion.$MinorVersion.* found"
 }
 
@@ -133,17 +119,29 @@ Write-Host "Copy Python binaries to $PythonArchPath"
 Copy-Item -Path ./$PythonExecName -Destination $PythonArchPath | Out-Null
 
 Write-Host "Install Python $Version in $PythonToolcachePath..."
-$ExecParams = Get-ExecParams -IsMSI $IsMSI -PythonArchPath $PythonArchPath
+$ExecParams = Get-ExecParams -IsMSI $IsMSI -IsFreeThreaded $IsFreeThreaded -PythonArchPath $PythonArchPath
 
 cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet"
-if ($LASTEXITCODE -ne 0)
-{
+if ($LASTEXITCODE -ne 0) {
     Throw "Error happened during Python installation"
 }
 
+if ($IsFreeThreaded) {
+    # Delete python.exe and create a symlink to free-threaded exe
+    Remove-Item -Path "$PythonArchPath\python.exe" -Force
+    New-Item -Path "$PythonArchPath\python.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python${MajorVersion}.${MinorVersion}t.exe"
+}
+
+Write-Host "Create `python3` symlink"
+New-Item -Path "$PythonArchPath\python3.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python.exe"
+
 Write-Host "Install and upgrade Pip"
+$Env:PIP_ROOT_USER_ACTION = "ignore"
 $PythonExePath = Join-Path -Path $PythonArchPath -ChildPath "python.exe"
-cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip"
+cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade --force-reinstall pip --no-warn-script-location"
+if ($LASTEXITCODE -ne 0) {
+    Throw "Error happened during pip installation / upgrade"
+}
 
 Write-Host "Create complete file"
 New-Item -ItemType File -Path $PythonVersionPath -Name "$Architecture.complete" | Out-Null

tests/ManifestConfig.Tests.ps1

@@ -0,0 +1,56 @@
+Import-Module (Join-Path $PSScriptRoot "../helpers/packages-generation/manifest-utils.psm1")
+
+$ConfigurationFile = Join-Path $PSScriptRoot "../config/python-manifest-config.json"
+$Configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
+
+$stableTestCases = @(
+
+    @{ ReleaseName = "python-3.13.0-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.13.0-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.13.0-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.13.0-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.13.0-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} },
+    @{ ReleaseName = "python-3.13.0-darwin-x64-freethreaded.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64-freethreaded"} },
+    @{ ReleaseName = "python-3.13.0-linux-20.04-x64-freethreaded.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64-freethreaded"} },
+    @{ ReleaseName = "python-3.13.0-linux-22.04-x64-freethreaded.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64-freethreaded"} },
+    @{ ReleaseName = "python-3.13.0-win32-x64-freethreaded.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64-freethreaded"} },
+    @{ ReleaseName = "python-3.13.0-win32-x86-freethreaded.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86-freethreaded"} }
+
+) | ForEach-Object { $_.Configuration = $Configuration; $_ }
+
+$unstableTestCases = @(
+
+   @{ ReleaseName = "python-3.14.0-alpha.5-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.14.0-alpha.5-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.14.0-alpha.5-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.14.0-alpha.5-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.14.0-alpha.5-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} },
+    @{ ReleaseName = "python-3.14.0-alpha.5-darwin-x64-freethreaded.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64-freethreaded"} },
+    @{ ReleaseName = "python-3.14.0-alpha.5-linux-20.04-x64-freethreaded.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64-freethreaded"} },
+    @{ ReleaseName = "python-3.14.0-alpha.5-linux-22.04-x64-freethreaded.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64-freethreaded"} },
+    @{ ReleaseName = "python-3.14.0-alpha.5-win32-x64-freethreaded.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64-freethreaded"} },
+    @{ ReleaseName = "python-3.14.0-alpha.5-win32-x86-freethreaded.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86-freethreaded"} }
+
+) | ForEach-Object { $_.Configuration = $Configuration; $_ }
+
+Describe "Python manifest config" {
+    Context "Stable versions" {
+        It "<ReleaseName>" -TestCases $stableTestCases {
+            $Release = @{ name = $ReleaseName }
+            $asset = New-AssetItem -ReleaseAsset $Release -Configuration $Configuration
+            $asset.platform | Should -Be $ExpectedResult.platform
+            $asset.platform_version | Should -Be $ExpectedResult.platform_version
+            $asset.arch | Should -Be $ExpectedResult.arch
+        }
+    }
+
+    Context "Prerelease versions" {
+        It "<ReleaseName>" -TestCases $unstableTestCases {
+            $Release = @{ name = $ReleaseName }
+            $asset = New-AssetItem -ReleaseAsset $Release -Configuration $Configuration
+            $asset.platform | Should -Be $ExpectedResult.platform
+            $asset.platform_version | Should -Be $ExpectedResult.platform_version
+            $asset.arch | Should -Be $ExpectedResult.arch
+        }
+    }
+}

tests/clean-toolcache.ps1

@@ -1,11 +0,0 @@
-if ($env:PLATFORM -match 'windows') {
-  $PythonFilter = "Name like '%Python%'"
-  Get-WmiObject Win32_Product -Filter $PythonFilter | Foreach-Object { 
-      Write-Host "Uninstalling $($_.Name) ..."
-      $_.Uninstall() | Out-Null 
-  }
-}
-
-$PythonToolcachePath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "Python"
-Write-Host "Removing Python toolcache directory ..."
-Remove-Item -Path $PythonToolcachePath -Recurse -Force

tests/python-tests.ps1

@@ -1,31 +1,43 @@
 param (
-    [Version] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    [semver] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
     $Version,
-    [String] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
-    $Platform
+    [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    $Platform,
+    [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    $Architecture
 )
 
+$HardwareArchitecture = $Architecture -replace "-freethreaded", ""
+
 Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
 Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1")
+Import-Module (Join-Path $PSScriptRoot "../builders/python-version.psm1")
 
-function Analyze-MissingModules([string] $buildOutputLocation) {
-    $searchStringStart = "Failed to build these modules:"
-    $searchStringEnd = "running build_scripts"
-    $pattern = "$searchStringStart(.*?)$searchStringEnd"
+BeforeAll {
+    function Analyze-MissingModules([string] $buildOutputLocation) {
+        $searchStringStart = "Failed to build these modules:"
+        $searchStringEnd = "running build_scripts"
+        $pattern = "$searchStringStart(.*?)$searchStringEnd"
 
-    $buildContent = Get-Content -Path $buildOutputLocation
-    $splitBuiltOutput = $buildContent -split "\n";
+        $buildContent = Get-Content -Path $buildOutputLocation
+        $splitBuiltOutput = $buildContent -split "\n";
 
-    ### Search for missing modules that are displayed between the search strings
-    $regexMatch = [regex]::match($SplitBuiltOutput, $Pattern)
-    if ($regexMatch.Success)
-    {
-        Write-Host "Failed missing modules:"
-        Write-Host $regexMatch.Groups[1].Value
-        return 1
+        ### Search for missing modules that are displayed between the search strings
+        $regexMatch = [regex]::match($SplitBuiltOutput, $Pattern)
+        if ($regexMatch.Success)
+        {
+            $module = $regexMatch.Groups[1].Value.Trim()
+            Write-Host "Failed missing modules:"
+            Write-Host $module
+            if ( ($module -eq "_tkinter") -and ( [semver]"$($Version.Major).$($Version.Minor)" -ge [semver]"3.10" -and $Version.PreReleaseLabel ) ) {
+                Write-Host "$module $Version ignored"
+            } else {
+                return 1
+            }
+        }
+
+        return 0
     }
-
-    return 0
 }
 
 Describe "Tests" {
@@ -33,7 +45,7 @@ Describe "Tests" {
         "python --version" | Should -ReturnZeroExitCode
         $pythonLocation = (Get-Command "python").Path
         $pythonLocation | Should -Not -BeNullOrEmpty
-        $expectedPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "Python"
+        $expectedPath = Join-Path -Path $env:RUNNER_TOOL_CACHE -ChildPath "Python"
         $pythonLocation.startsWith($expectedPath) | Should -BeTrue
     }
 
@@ -41,10 +53,28 @@ Describe "Tests" {
         "python ./sources/simple-test.py" | Should -ReturnZeroExitCode
     }
 
+    # linux has no display name and no $DISPLAY environment variable - skip tk test
+    # if (-not (($Platform -match "ubuntu") -or ($Platform -match "linux"))) {
+    #     It "Check if tcl/tk has the same headed and library versions" {
+	#     "python ./sources/tcltk.py" | Should -ReturnZeroExitCode
+    #     }
+    # }
+
+    if (($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($HardwareArchitecture -ne "arm64"))) {
+        It "Check if sqlite3 module is installed" {
+            "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
+        }
+    }
+
+    It "Run pip" {
+        "pip install requests" | Should -ReturnZeroExitCode
+        "pip uninstall requests -y" | Should -ReturnZeroExitCode
+    }
+
     if (IsNixPlatform $Platform) {
 
         It "Check for failed modules in build_output" {
-            $buildOutputLocation = Join-Path $env:BUILD_BINARIESDIRECTORY "build_output.txt"
+            $buildOutputLocation = Join-Path $env:RUNNER_TEMP "build_output.txt"
             Analyze-MissingModules $buildOutputLocation | Should -Be 0
         }
 
@@ -53,7 +83,8 @@ Describe "Tests" {
         }
 
         It "Check if python configuration is correct" {
-            "python ./sources/python-config-test.py" | Should -ReturnZeroExitCode
+            $nativeVersion = Convert-Version -version $Version
+            "python ./sources/python-config-test.py $Version $nativeVersion $Architecture" | Should -ReturnZeroExitCode
         }
 
         It "Check if shared libraries are linked correctly" {
@@ -61,12 +92,11 @@ Describe "Tests" {
         }
     }
 
-    # Pyinstaller 3.5 does not support Python 3.8.0. Check issue https://github.com/pyinstaller/pyinstaller/issues/4311
-    if ($Version -lt "3.8.0") {
-        It "Validate Pyinstaller" {
-            "pip install pyinstaller" | Should -ReturnZeroExitCode
-            "pyinstaller --onefile ./sources/simple-test.py" | Should -ReturnZeroExitCode
-            "./dist/simple-test" | Should -ReturnZeroExitCode
-        }
+    It "Check urlopen with HTTPS works" {
+        "python ./sources/python-urlopen-https.py" | Should -ReturnZeroExitCode
     }
-}
+
+    It "Check a single dist-info per distribution is present" {
+        "python ./sources/dist-info.py" | Should -ReturnZeroExitCode
+    }
+}

tests/sources/dist-info.py

@@ -0,0 +1,24 @@
+import glob
+import os.path
+import sysconfig
+from collections import defaultdict
+
+
+def check_dist_info():
+    paths = set([sysconfig.get_path("purelib"), sysconfig.get_path("platlib")])
+    versions = defaultdict(list)
+    for path in paths:
+        pattern = os.path.join(path, "*.dist-info")
+        for dist_info in glob.glob(pattern):
+            name = os.path.basename(dist_info).split("-", maxsplit=1)[0]
+            versions[name].append(dist_info)
+    exit_code = 0
+    for name in versions:
+        if len(versions[name]) > 1:
+            print("multiple dist-info found for {}: {}".format(name, versions[name]))
+            exit_code = 1
+    exit(exit_code)
+
+
+if __name__ == "__main__":
+    check_dist_info()

tests/sources/python-config-output.py

@@ -1,6 +1,4 @@
-import distutils.sysconfig
 import sysconfig
 
 from pprint import pprint
 pprint(sysconfig.get_config_vars())
-pprint(distutils.sysconfig.get_config_vars())

tests/sources/python-config-test.py

@@ -1,4 +1,3 @@
-import distutils.sysconfig
 import sysconfig
 import sys
 import platform
@@ -6,18 +5,36 @@ import os
 
 # Define variables
 os_type = platform.system()
-version = sys.version.split(" ")[0]
+version = sys.argv[1]
+nativeVersion = sys.argv[2]
+architecture = sys.argv[3]
+hw_architecture = architecture.replace('-freethreaded', '')
+
+versions=version.split(".")
+version_major=int(versions[0])
+version_minor=int(versions[1])
+
+pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (hw_architecture == "arm64"))
 
 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
 
 is_shared = sysconfig.get_config_var('Py_ENABLE_SHARED')
 have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE")
+is_free_threaded = sysconfig.get_config_var('Py_GIL_DISABLED')
 
 ### Define expected variables
 if os_type == 'Linux': expected_ld_library_extension = 'so'
 if os_type == 'Darwin': expected_ld_library_extension = 'dylib'
-expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version)
+if is_free_threaded:
+    framework_name = 'PythonT.framework'
+else:
+    framework_name = 'Python.framework'
+
+if pkg_installer:
+    expected_lib_dir_path = f'/Library/Frameworks/{framework_name}/Versions/{version_major}.{version_minor}/lib'
+else:
+    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/{architecture}/lib'
 
 # Check modules
 ### Validate libraries path
@@ -28,7 +45,7 @@ if lib_dir_path != expected_lib_dir_path:
 ### Validate shared libraries
 if is_shared:
     print('%s was built with shared extensions' % ld_library_name)
-    
+
     ### Validate libpython extension
     ld_library_extension = ld_library_name.split('.')[-1]
     if ld_library_extension != expected_ld_library_extension:
@@ -36,33 +53,37 @@ if is_shared:
         exit(1)
 else:
     print('%s was built without shared extensions' % ld_library_name)
-    exit(1)
+    if not pkg_installer:
+        exit(1)
 
 ### Validate macOS
 if os_type == 'Darwin':
     ### Validate openssl links
-    if version < "3.7.0":
-        expected_ldflags = '-L/usr/local/opt/openssl@1.1/lib'
+    if version_major == 3 and version_minor < 7:
+        expected_ldflags = '-L/usr/local/opt/openssl@3/lib'
         ldflags = sysconfig.get_config_var('LDFLAGS')
 
         if not expected_ldflags in ldflags:
             print('Invalid ldflags: %s; Expected: %s' % (ldflags, expected_ldflags))
             exit(1)
     else:
-        expected_openssl_includes = '-I/usr/local/opt/openssl/include'
-        expected_openssl_ldflags ='-L/usr/local/opt/openssl/lib'
-        
+        expected_openssl_includes = '-I/usr/local/opt/openssl@3/include'
+        expected_openssl_ldflags ='-L/usr/local/opt/openssl@3/lib'
+
         openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES')
         openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS')
 
         if openssl_includes != expected_openssl_includes:
             print('Invalid openssl_includes: %s; Expected: %s' % (openssl_includes, expected_openssl_includes))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
         if openssl_ldflags != expected_openssl_ldflags:
             print('Invalid openssl_ldflags: %s; Expected: %s' % (openssl_ldflags, expected_openssl_ldflags))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
 
 ### Validate libreadline
-if not have_libreadline:
-    print('Missing libreadline')
-    exit(1)
+if sys.version_info < (3, 12):
+    if not have_libreadline:
+        print('Missing libreadline')
+        exit(1)

tests/sources/python-modules.py

@@ -3,12 +3,10 @@ Make sure all the optional modules are installed.
 This is needed for Linux since we build from source.
 """
 
-from __future__ import print_function
-
 import importlib
 import sys
 
-# The Python standard library as of Python 3.0
+# The Python standard library as of Python 3.9
 standard_library = [
     'abc',
     'aifc',
@@ -17,6 +15,7 @@ standard_library = [
     'ast',
     'asynchat',
     'asyncore',
+    'asyncio',
     'base64',
     'bdb',
     'binhex',
@@ -34,14 +33,17 @@ standard_library = [
     'collections',
     'colorsys',
     'compileall',
+    'concurrent',
     'configparser',
     'contextlib',
+    'contextvars',
     'copy',
     'copyreg',
     'crypt',
     'csv',
     'ctypes',
     'curses',
+    'dataclasses',
     'datetime',
     'dbm',
     'decimal',
@@ -49,9 +51,11 @@ standard_library = [
     'dis',
     'distutils',
     'doctest',
-    'dummy_threading',
     'email',
     'encodings',
+    'ensurepip',
+    'enum',
+    'faulthandler',
     'filecmp',
     'fileinput',
     'fnmatch',
@@ -77,13 +81,14 @@ standard_library = [
     'importlib',
     'inspect',
     'io',
+    'ipaddress',
     'json',
     'keyword',
     'lib2to3',
     'linecache',
     'locale',
     'logging',
-    'macpath',
+    'lzma',
     'mailbox',
     'mailcap',
     'mimetypes',
@@ -98,6 +103,7 @@ standard_library = [
     'operator',
     'optparse',
     'os',
+    'pathlib',
     'pdb',
     'pickle',
     'pickletools',
@@ -124,6 +130,8 @@ standard_library = [
     'rlcompleter',
     'runpy',
     'sched',
+    'secrets',
+    'selectors',
     'shelve',
     'shlex',
     'shutil',
@@ -139,7 +147,9 @@ standard_library = [
     'sre_constants',
     'sre_parse',
     'ssl',
+    '_ssl',
     'stat',
+    'statistics',
     'string',
     'stringprep',
     'struct',
@@ -162,14 +172,17 @@ standard_library = [
     'tokenize',
     'trace',
     'traceback',
+    'tracemalloc',
     'tty',
     'turtle',
     'turtledemo',
     'types',
+    'typing',
     'unittest',
     'urllib',
     'uu',
     'uuid',
+    'venv',
     'warnings',
     'wave',
     'weakref',
@@ -178,79 +191,56 @@ standard_library = [
     'xdrlib',
     'xml',
     'xmlrpc',
+    'zipapp',
     'zipfile'
 ]
 
-# Modules that had different names in Python 2
-if sys.version_info.major == 2:
-    def replace(lst, old, new):
-        lst[lst.index(old)] = new
-
-    # Keys are the Python 2 names
-    # Values are the Python 3 names
-    renames = {
-        'ConfigParser': 'configparser',
-        'copy_reg': 'copyreg',
-        'HTMLParser': 'html',
-        'httplib': 'http',
-        'Queue': 'queue',
-        'repr': 'reprlib',
-        'SocketServer': 'socketserver',
-        'xmlrpclib': 'xmlrpc',
-        'Tkinter': 'tkinter'
-    }
-
-    # All of the Python 3 names should be in the list
-    for python2name, python3name in renames.items():
-        replace(standard_library, python3name, python2name)
-
 # Add new modules
 # See https://docs.python.org/3/whatsnew/index.html
-if sys.version_info >= (3, 2):
-    standard_library.extend([
-        'concurrent',
-    ])
+# 'symbol' and 'formatter' modules have been removed from Python 3.10
+if sys.version_info >= (3, 10):
+    standard_library.remove('symbol')
+    standard_library.remove('formatter')
 
-if sys.version_info >= (3, 3):
-    standard_library.extend([
-        'ipaddress',
-        'faulthandler',
-        'lzma',
-        'venv',
-    ])
+# 'binhex' module has been removed from Python 3.11
+if sys.version_info >= (3, 11):
+    standard_library.remove('binhex')
 
-if sys.version_info >= (3, 4):
-    standard_library.extend([
-        'asyncio',
-        'ensurepip',
-        'enum',
-        'pathlib',
-        'selectors',
-        'statistics',
-        'tracemalloc',
-    ])
+# 'smtpd', 'asyncore' and 'asynchat' modules have been removed from Python 3.12
+# https://docs.python.org/dev/whatsnew/3.12.html
+if sys.version_info >= (3, 12):
+    standard_library.remove('distutils')
+    standard_library.remove('imp')
+    standard_library.remove('smtpd')
+    standard_library.remove('asyncore')
+    standard_library.remove('asynchat')
 
-if sys.version_info >= (3, 5):
-    standard_library.extend([
-        'typing',
-        'zipapp',
-    ])
+# 'aifc', 'cgi', 'cgitb', 'chunk', 'crypt', 'imghdr', 'lib2to3', 'mailcap', 'nntplib',
+# 'pipes', 'sndhdr', 'sunau', 'telnetlib', 'uu' and 'xdrlib' modules have been removed
+# from Python 3.13
+# https://docs.python.org/dev/whatsnew/3.13.html
+if sys.version_info >= (3, 13):
+    standard_library.remove('aifc')
+    standard_library.remove('cgi')
+    standard_library.remove('cgitb')
+    standard_library.remove('chunk')
+    standard_library.remove('crypt')
+    standard_library.remove('imghdr')
+    standard_library.remove('lib2to3')
+    standard_library.remove('mailcap')
+    standard_library.remove('nntplib')
+    standard_library.remove('pipes')
+    standard_library.remove('sndhdr')
+    standard_library.remove('sunau')
+    standard_library.remove('telnetlib')
+    standard_library.remove('uu')
+    standard_library.remove('xdrlib')
 
-if sys.version_info >= (3, 6):
+if sys.version_info >= (3, 14):
     standard_library.extend([
-        'secrets',
+        'annotationlib',
     ])
 
-if sys.version_info >= (3, 7):
-    standard_library.extend([
-        'contextvars',
-        'dataclasses',
-    ])
-
-# 'macpath' module has been removed from Python 3.8
-if sys.version_info > (3, 7):
-    standard_library.remove('macpath')
-
 # Remove tkinter and Easter eggs
 excluded_modules = [
     'antigravity',
@@ -261,7 +251,6 @@ excluded_modules = [
 def check_missing_modules(expected_modules):
     missing = []
     for module in expected_modules:
-        print('Try to import module ', module)
         try:
             importlib.import_module(module)
         except:

tests/sources/python-sqlite3.py

@@ -0,0 +1,19 @@
+import sqlite3
+from sqlite3 import Error
+
+def create_connection(db_file):
+    """ create a database connection to a SQLite database """
+    conn = None
+    try:
+        print('Sqlite3 version: ', sqlite3.version)
+        conn = sqlite3.connect(db_file)
+        conn.enable_load_extension(True)
+    except Error as e:
+        print(e)
+        exit(1)
+    finally:
+        if conn:
+            conn.close()
+
+if __name__ == '__main__':
+    create_connection(r"pythonsqlite.db")

tests/sources/python-urlopen-https.py

@@ -0,0 +1,6 @@
+import sys
+from urllib.request import urlopen
+
+response = urlopen("https://raw.githubusercontent.com/actions/python-versions/c641695f6a07526c18f10e374e503e649fef9427/.gitignore")
+data = response.read()
+assert len(data) == 140, len(data)

tests/sources/tcltk.py

@@ -0,0 +1,10 @@
+import tkinter
+import _tkinter
+
+header = _tkinter.TK_VERSION
+lib = tkinter.Tk().getvar('tk_version')
+
+if lib != header:
+  print('header version=' + header)
+  print('lib version=' + lib)
+  exit(1)