Update versions-manifest (#335)

Co-authored-by: Service account <no-reply@microsoft.com>

.github/workflows/build-python-packages.yml

@@ -1,21 +1,26 @@
 name: Build Python package
-run-name: Generate Python ${{ inputs.VERSION || '3.11.0' }}
+run-name: Generate Python ${{ inputs.VERSION || '3.12.3' }}
 on:
   workflow_dispatch:
     inputs:
       VERSION:
         description: 'Python version to build and upload'
-        default: '3.11.0'
+        default: '3.12.3'
         required: true
       PUBLISH_RELEASES:
         description: 'Whether to publish releases'
         required: true
         type: boolean
         default: false
+      THREADING_BUILD_MODES:
+        description: 'CPython threading build modes'
+        required: true
+        type: string
+        default: 'default,freethreaded'
       PLATFORMS:
         description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
         required: true
-        default: 'ubuntu-20.04,ubuntu-22.04,macos-11_x64,macos-11_arm64,windows-2019_x64,windows-2019_x86'
+        default: 'ubuntu-20.04,ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,ubuntu-24.04_arm64,macos-13_x64,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-2019_arm64'
   pull_request:
     paths-ignore:
     - 'versions-manifest.json'
@@ -25,7 +30,7 @@ on:
     - 'main'
 
 env:
-  VERSION: ${{ inputs.VERSION || '3.11.0' }}
+  VERSION: ${{ inputs.VERSION || '3.12.3' }}
 defaults:
   run:
     shell: pwsh
@@ -39,26 +44,47 @@ jobs:
       - name: Generate execution matrix
         id: generate-matrix
         run: |
-          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,macos-11,macos-11_arm64,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
+          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,ubuntu-24.04_arm64,macos-13,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-2019_arm64' }}".Split(",").Trim()
+          [String[]]$buildModes = "${{ inputs.threading_build_modes || 'default' }}".Split(",").Trim()
           $matrix = @()
-
+          
           foreach ($configuration in $configurations) {
-            $parts = $configuration.Split("_")
-            $os = $parts[0]
-            $arch = if ($parts[1]) {$parts[1]} else {"x64"}
-            switch -wildcard ($os) {
-              "*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
-              "*macos*" { $platform = 'darwin' }
-              "*windows*" { $platform = 'win32' }
-            }
-            $matrix += @{
-              'platform' = $platform
-              'os' = $os
-              'arch' = $arch
+            foreach ($buildMode in $buildModes) {
+              $parts = $configuration.Split("_")
+              $os = $parts[0]
+              $arch = if ($parts[1]) {$parts[1]} else {"x64"}
+              switch -wildcard ($os) {
+                "*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
+                "*macos*" { $platform = 'darwin' }
+                "*windows*" { $platform = 'win32' }
+              }
+
+              if ($configuration -eq "ubuntu-22.04_arm64") {
+                $os = "setup-actions-ubuntu-arm64-2-core"
+              }
+              elseif ($configuration -eq "ubuntu-24.04_arm64") {
+                $os = "setup-actions-ubuntu24-arm64-2-core"
+              }
+              elseif ($configuration -eq "windows-2019_arm64") {
+                $os = "setup-actions-windows-arm64-4-core"
+              }
+
+              if ($buildMode -eq "freethreaded") {
+                if ([semver]"${{ inputs.VERSION }}" -lt [semver]"3.13.0") {
+                  continue;
+                }
+                $arch += "-freethreaded"
+              }
+
+              $matrix += @{
+                'platform' = $platform
+                'os' = $os
+                'arch' = $arch
+              }
             }
           }
           echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT
-
+          
   build_python:
     needs: generate_matrix
     strategy:
@@ -67,24 +93,52 @@ jobs:
         include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
     runs-on: ${{ matrix.os }}
     env:
-      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.12.3' }}-${{ matrix.platform }}-${{ matrix.arch }}
     steps:
+      - name: Setup Environment on Windows ARM64 Runner
+        if: matrix.os == 'setup-actions-windows-arm64-4-core'
+        shell: powershell
+        run: |
+            # Install Chocolatey
+            Set-ExecutionPolicy Bypass -Scope Process -Force
+            [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
+            iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
+            echo "C:\ProgramData\Chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+        
+            # Install PowerShell
+            choco install powershell-core -y
+            echo "C:\Program Files\PowerShell\7" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+        
+            # Install Git
+            choco install git -y
+            echo "C:\Program Files\Git\cmd" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+        
+            
+             # Install 7-Zip
+            choco install 7zip -y
+            echo "C:\ProgramData\chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+            
 
       - name: Check out repository code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
 
+      - name: Adding Symlink
+        if: matrix.os == 'macos-13'
+        run: ln -sf /usr/local/opt/tcl-tk@8 /usr/local/opt/tcl-tk
+
       - name: Build Python ${{ env.VERSION }}
         run: |
           ./builders/build-python.ps1 -Version $env:VERSION `
                   -Platform ${{ matrix.platform }} -Architecture ${{ matrix.arch }}
-
+                  
       - name: Publish artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ env.ARTIFACT_NAME }}
           path: ${{ runner.temp }}/artifact
+          if-no-files-found: error
 
   test_python:
     needs: [generate_matrix, build_python]
@@ -94,21 +148,49 @@ jobs:
         include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
     runs-on: ${{ matrix.os }}
     env:
-      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.12.3' }}-${{ matrix.platform }}-${{ matrix.arch }}
     steps:
-
+      - name: Setup Environment on Windows ARM64 Runner
+        if: matrix.os == 'setup-actions-windows-arm64-4-core'
+        shell: powershell
+        run: |
+            # Install Chocolatey
+            Set-ExecutionPolicy Bypass -Scope Process -Force
+            [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
+            iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
+            echo "C:\ProgramData\Chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+        
+            # Install PowerShell
+            choco install powershell-core -y
+            echo "C:\Program Files\PowerShell\7" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+        
+            # Install Git
+            choco install git -y
+            echo "C:\Program Files\Git\cmd" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+        
+            
+             # Install 7-Zip
+            choco install 7zip -y
+            echo "C:\ProgramData\chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
+            
       - name: Check out repository code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
 
       - name: Fully cleanup the toolcache directory before testing
         run: ./helpers/clean-toolcache.ps1 -ToolName "Python"
 
+      - name: Delete macOS /Library/Frameworks/Python.framework
+        if: matrix.platform == 'darwin'
+        shell: bash
+        run: if [ -d /Library/Frameworks/Python.framework ]; then sudo rm -rf /Library/Frameworks/Python.framework; fi
+
       - name: Download artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          path: ${{ runner.temp }}
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
 
       - name: Extract files
         run: |
@@ -123,6 +205,7 @@ jobs:
 
       - name: Apply build artifact to the local machine
         run: |
+          $PSNativeCommandUseErrorActionPreference = $true
           if ('${{ matrix.platform }}' -eq 'win32') {
             powershell ./setup.ps1
             } else {
@@ -132,11 +215,14 @@ jobs:
         working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
 
       - name: Setup Python ${{ env.VERSION }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.VERSION }}
           architecture: ${{ matrix.arch }}
 
+      - name: Python version
+        run: python -VVV
+
       - name: Verbose sysconfig dump
         if: runner.os == 'Linux' || runner.os == 'macOS'
         run: python ./sources/python-config-output.py
@@ -150,7 +236,7 @@ jobs:
 
       - name: Run tests
         run: |
-            Install-Module Pester -Force -Scope CurrentUser
+            Install-Module Pester -Force -Scope CurrentUser -SkipPublisherCheck
             Import-Module Pester
             $pesterContainer = New-PesterContainer -Path './python-tests.ps1' -Data @{
                 Version="${{ env.VERSION }}";
@@ -170,7 +256,7 @@ jobs:
       needs: test_python
       runs-on: ubuntu-latest
       steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
 
       - name: Publish Release ${{ env.VERSION }}
         id: create_release
@@ -196,7 +282,7 @@ jobs:
           } 
   
       - name: Upload release assets
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
@@ -220,7 +306,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Trigger "Create Pull Request" workflow
-      uses: actions/github-script@v6
+      uses: actions/github-script@v7
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
         script: |
@@ -230,4 +316,3 @@ jobs:
             workflow_id: 'create-pr.yml',
             ref: 'main'
           });
-

.github/workflows/codeql-analysis.yml

@@ -13,4 +13,5 @@ jobs:
     name: CodeQL analysis 
     uses: actions/reusable-workflows/.github/workflows/codeql-analysis.yml@main
     with:
-     languages: "['python']"
+     languages: '["python"]'
+   

.github/workflows/manifest-config-validation.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: true
 

.github/workflows/releases-validation.yml

@@ -14,11 +14,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04]
-        python: [3.9.12, 3.10.8, 3.11.8]
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04, macos-13, ubuntu-latest]
+        python: [3.9.13, 3.10.11, 3.11.8, 3.12.7, 3.13.0]
     steps:
     - name: setup-python ${{ matrix.python }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python }}
 

.github/workflows/test-python-version.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04] 
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04, ubuntu-latest, macos-13] 
     steps:
     - name: Setup Python ${{ github.event.inputs.version }}
       uses: actions/setup-python@main

builders/macos-python-builder.psm1

@@ -68,10 +68,10 @@ class macOSPythonBuilder : NixPythonBuilder {
         ### and then add the appropriate paths for the header and library files to configure command.
         ### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies)
         if ($this.Version -lt "3.7.0") {
-            $env:LDFLAGS = "-L/usr/local/opt/openssl@1.1/lib -L/usr/local/opt/zlib/lib"
-            $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/zlib/include"
+            $env:LDFLAGS = "-L/usr/local/opt/openssl@3/lib -L/usr/local/opt/zlib/lib"
+            $env:CFLAGS = "-I/usr/local/opt/openssl@3/include -I/usr/local/opt/zlib/include"
         } else {
-            $configureString += " --with-openssl=/usr/local/opt/openssl@1.1"
+            $configureString += " --with-openssl=/usr/local/opt/openssl@3"
 
             # For Python 3.7.2 and 3.7.3 we need to provide PATH for zlib to pack it properly. Otherwise the build will fail
             # with the error: zipimport.ZipImportError: can't decompress data; zlib not available
@@ -80,9 +80,9 @@ class macOSPythonBuilder : NixPythonBuilder {
                 $env:CFLAGS = "-I/usr/local/opt/zlib/include"
             }
 
-            if ($this.Version -gt "3.7.12") {
-                $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'"
-	        }
+            # if ($this.Version -gt "3.7.12") {
+                $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include/tcl-tk' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'"
+            # }
 
             if ($this.Version -eq "3.7.17") {
                 $env:LDFLAGS += " -L$(brew --prefix bzip2)/lib -L$(brew --prefix readline)/lib -L$(brew --prefix ncurses)/lib"
@@ -151,6 +151,37 @@ class macOSPythonBuilder : NixPythonBuilder {
         return $pkgLocation
     }
 
+    [string] GetFrameworkName() {
+        <#
+        .SYNOPSIS
+        Get the Python installation Package name.
+        #>
+
+        if ($this.IsFreeThreaded()) {
+            return "PythonT.framework"
+        } else {
+            return "Python.framework"
+        }
+    }
+
+    [string] GetPkgChoices() {
+        <#
+        .SYNOPSIS
+        Reads the configuration XML file for the Python installer
+        #>
+
+        $config = if ($this.IsFreeThreaded()) { "freethreaded" } else { "default" }
+        $choicesFile = Join-Path $PSScriptRoot "../config/macos-pkg-choices-$($config).xml"
+        $choicesTemplate = Get-Content -Path $choicesFile -Raw
+
+        $variablesToReplace = @{
+            "{{__VERSION_MAJOR_MINOR__}}" = "$($this.Version.Major).$($this.Version.Minor)";
+        }
+
+        $variablesToReplace.keys | ForEach-Object { $choicesTemplate = $choicesTemplate.Replace($_, $variablesToReplace[$_]) }
+        return $choicesTemplate
+    }
+
     [void] CreateInstallationScriptPkg() {
         <#
         .SYNOPSIS
@@ -165,6 +196,8 @@ class macOSPythonBuilder : NixPythonBuilder {
             "{{__VERSION_FULL__}}" = $this.Version;
             "{{__PKG_NAME__}}" = $this.GetPkgName();
             "{{__ARCH__}}" = $this.Architecture;
+            "{{__FRAMEWORK_NAME__}}" = $this.GetFrameworkName();
+            "{{__PKG_CHOICES__}}" = $this.GetPkgChoices();
         }
 
         $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }

builders/nix-python-builder.psm1

@@ -97,6 +97,7 @@ class NixPythonBuilder : PythonBuilder {
 
         $variablesToReplace = @{
             "{{__VERSION_FULL__}}" = $this.Version;
+            "{{__ARCH__}}" = $this.Architecture;
         }
         $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
 
@@ -114,7 +115,7 @@ class NixPythonBuilder : PythonBuilder {
         Write-Debug "make Python $($this.Version)-$($this.Architecture) $($this.Platform)"
         $buildOutputLocation = New-Item -Path $this.WorkFolderLocation -Name "build_output.txt" -ItemType File
         
-        Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue	
+        Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue
         Execute-Command -Command "make install" -ErrorAction Continue
 
         Write-Debug "Done; Make log location: $buildOutputLocation"

builders/python-builder.psm1

@@ -94,6 +94,24 @@ class PythonBuilder {
         return "$($this.Version.Major).$($this.Version.Minor).$($this.Version.Patch)"
     }
 
+    [string] GetHardwareArchitecture() {
+        <#
+        .SYNOPSIS
+        The hardware architecture (x64, arm64) without any Python free threading suffix.
+        #>
+
+        return $this.Architecture.Replace("-freethreaded", "")
+    }
+
+    [bool] IsFreeThreaded() {
+        <#
+        .SYNOPSIS
+        Check if Python version is free threaded.
+        #>
+
+        return $this.Architecture.EndsWith("-freethreaded")
+    }
+
     [void] PreparePythonToolcacheLocation() {
         <#
         .SYNOPSIS

builders/ubuntu-python-builder.psm1

@@ -37,16 +37,17 @@ class UbuntuPythonBuilder : NixPythonBuilder {
         $configureString += " --enable-shared"
         $configureString += " --enable-optimizations"
 
-        ### Compile with ucs4 for Python 2.x. On 3.x, ucs4 is enabled by default
-        if ($this.Version -lt "3.0.0") {
-            $configureString += " --enable-unicode=ucs4"
+        if ($this.IsFreeThreaded()) {
+            if ($this.Version -lt "3.13.0") {
+                Write-Host "Python versions lower than 3.13.0 do not support free threading"
+                exit 1
+            }
+            $configureString += " --disable-gil"
         }
 
-        ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
+        ### Compile with support of loadable sqlite extensions.
         ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
-        if ($this.Version -ge "3.2.0") {
-            $configureString += " --enable-loadable-sqlite-extensions"
-        }
+        $configureString += " --enable-loadable-sqlite-extensions"
 
         Write-Host "The passed configure options are: "
         Write-Host $configureString
@@ -60,17 +61,13 @@ class UbuntuPythonBuilder : NixPythonBuilder {
         Prepare system environment by installing dependencies and required packages.
         #>
 
-        if (($this.Version -gt "3.0.0") -and ($this.Version -lt "3.5.3")) {
-            Write-Host "Python3 versions lower than 3.5.3 are not supported"
+        if ($this.Version -lt "3.5.3") {
+            Write-Host "Python versions lower than 3.5.3 are not supported"
             exit 1
         }
 
         ### Compile with tkinter support
-        if ($this.Version -gt "3.0.0") {
-            $tkinterInstallString = "sudo apt-get install -y --allow-downgrades python3-tk tk-dev"
-        } else {
-            $tkinterInstallString = "sudo apt install -y python-tk tk-dev"
-        }
+        $tkinterInstallString = "sudo apt-get install -y --allow-downgrades python3-tk tk-dev"
 
         Execute-Command -Command "sudo apt-get update"
         Execute-Command -Command $tkinterInstallString
@@ -85,7 +82,8 @@ class UbuntuPythonBuilder : NixPythonBuilder {
             "libsqlite3-dev",
             "libncursesw5-dev",
             "libreadline-dev",
-            "libgdbm-dev"
+            "libgdbm-dev",
+            "liblzma-dev"
         ) | ForEach-Object {
             Execute-Command -Command "sudo apt install -y $_"
         }

builders/win-python-builder.psm1

@@ -54,12 +54,14 @@ class WinPythonBuilder : PythonBuilder {
         #>
 
         $ArchitectureExtension = ""
-        if ($this.Architecture -eq "x64") {
+        if ($this.GetHardwareArchitecture() -eq "x64") {
             if ($this.Version -ge "3.5") {
                 $ArchitectureExtension = "-amd64"
             } else {
                 $ArchitectureExtension = ".amd64"
             }
+        } elseif ($this.GetHardwareArchitecture() -eq "arm64") {
+                $ArchitectureExtension = "-arm64"
         }
 
         return $ArchitectureExtension
@@ -111,6 +113,7 @@ class WinPythonBuilder : PythonBuilder {
 
         $variablesToReplace = @{
             "{{__ARCHITECTURE__}}" = $this.Architecture;
+            "{{__HARDWARE_ARCHITECTURE__}}" = $this.GetHardwareArchitecture();
             "{{__VERSION__}}" = $this.Version;
             "{{__PYTHON_EXEC_NAME__}}" = $pythonExecName
         }

config/macos-pkg-choices-default.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<array>
+    <dict>
+   </dict>
+</array>
+</plist>

config/macos-pkg-choices-freethreaded.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<array>
+    <dict>
+        <key>attributeSetting</key>
+        <integer>1</integer>
+        <key>choiceAttribute</key>
+        <string>selected</string>
+        <key>choiceIdentifier</key>
+        <string>org.python.Python.PythonTFramework-{{__VERSION_MAJOR_MINOR__}}</string>
+    </dict>
+</array>
+</plist>

config/python-manifest-config.json

@@ -1,5 +1,5 @@
 {
-    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+)",
+    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+(-freethreaded)?)",
     "groups": {
         "arch": 4,
         "platform": 2,

installers/macos-pkg-setup-template.sh

@@ -2,6 +2,11 @@ set -e
 
 PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
 PYTHON_PKG_NAME="{{__PKG_NAME__}}"
+PYTHON_FRAMEWORK_NAME="{{__FRAMEWORK_NAME__}}"
+PYTHON_PKG_CHOICES=$(cat << 'EOF'
+{{__PKG_CHOICES__}}
+EOF
+)
 ARCH="{{__ARCH__}}"
 MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
 MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
@@ -20,7 +25,7 @@ fi
 PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
 PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
 PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
-PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
+PYTHON_FRAMEWORK_PATH="/Library/Frameworks/${PYTHON_FRAMEWORK_NAME}/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
 PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
 
 echo "Check if Python hostedtoolcache folder exist..."
@@ -38,8 +43,11 @@ else
     done
 fi
 
+PYTHON_PKG_CHOICES_FILES=$(mktemp)
+echo "$PYTHON_PKG_CHOICES" > $PYTHON_PKG_CHOICES_FILES
+
 echo "Install Python binaries from prebuilt package"
-sudo installer -pkg $PYTHON_PKG_NAME -target /
+sudo installer -pkg $PYTHON_PKG_NAME -applyChoiceChangesXML $PYTHON_PKG_CHOICES_FILES -target /
 
 echo "Create hostedtoolcach symlinks (Required for the backward compatibility)"
 echo "Create Python $PYTHON_FULL_VERSION folder"
@@ -53,7 +61,9 @@ ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib
 
 echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
 ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
+chmod +x python
 
+# Note that bin is a symlink so referencing .. from bin will not work as expected
 cd bin/
 
 # This symlink already exists if Python version with the same major.minor version is installed,
@@ -62,15 +72,20 @@ if [ ! -f $PYTHON_MAJOR_MINOR ]; then
     ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
 fi
 
+if [ ! -f $PYTHON_MAJOR ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR
+fi
+
 if [ ! -f python ]; then
     ln -s $PYTHON_MAJOR_DOT_MINOR python
 fi
 
-chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
+chmod +x $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
 
 echo "Upgrading pip..."
+export PIP_ROOT_USER_ACTION=ignore
 ./python -m ensurepip
-./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location --root-user-action=ignore
+./python -m pip install --upgrade --force-reinstall pip --disable-pip-version-check --no-warn-script-location
 
 echo "Install OpenSSL certificates"
 sh -e "${PYTHON_APPLICATION_PATH}/Install Certificates.command"

installers/nix-setup-template.sh

@@ -1,6 +1,7 @@
 set -e
 
 PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
+ARCH="{{__ARCH__}}"
 MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
 MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
 
@@ -17,15 +18,15 @@ fi
 
 PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
 PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
-PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
 
 echo "Check if Python hostedtoolcache folder exist..."
 if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
     echo "Creating Python hostedtoolcache folder..."
     mkdir -p $PYTHON_TOOLCACHE_PATH
-elif [ -d $PYTHON_TOOLCACHE_VERSION_PATH ]; then
-    echo "Deleting Python $PYTHON_FULL_VERSION"
-    rm -rf $PYTHON_TOOLCACHE_VERSION_PATH
+elif [ -d $PYTHON_TOOLCACHE_VERSION_ARCH_PATH ]; then
+    echo "Deleting Python $PYTHON_FULL_VERSION ($ARCH)"
+    rm -rf $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
 fi
 
 echo "Create Python $PYTHON_FULL_VERSION folder"
@@ -49,8 +50,9 @@ fi
 chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR python
 
 echo "Upgrading pip..."
+export PIP_ROOT_USER_ACTION=ignore
 ./python -m ensurepip
-./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location --root-user-action=ignore
+./python -m pip install --upgrade --force-reinstall pip --disable-pip-version-check --no-warn-script-location
 
 echo "Create complete file"
-touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete
+touch $PYTHON_TOOLCACHE_VERSION_PATH/$ARCH.complete

installers/win-setup-template.ps1

@@ -1,4 +1,5 @@
 [String] $Architecture = "{{__ARCHITECTURE__}}"
+[String] $HardwareArchitecture = "{{__HARDWARE_ARCHITECTURE__}}"
 [String] $Version = "{{__VERSION__}}"
 [String] $PythonExecName = "{{__PYTHON_EXEC_NAME__}}"
 
@@ -25,7 +26,7 @@ function Remove-RegistryEntries {
         [Parameter(Mandatory)][Int32] $MinorVersion
     )
 
-    $versionFilter = Get-RegistryVersionFilter -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
+    $versionFilter = Get-RegistryVersionFilter -Architecture $HardwareArchitecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
 
     $regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products"
     if (Test-Path -Path Registry::$regPath) {
@@ -61,13 +62,15 @@ function Remove-RegistryEntries {
 function Get-ExecParams {
     param(
         [Parameter(Mandatory)][Boolean] $IsMSI,
+        [Parameter(Mandatory)][Boolean] $IsFreeThreaded,
         [Parameter(Mandatory)][String] $PythonArchPath
     )
 
     if ($IsMSI) {
         "TARGETDIR=$PythonArchPath ALLUSERS=1"
     } else {
-        "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1"
+        $Include_freethreaded = if ($IsFreeThreaded) { "Include_freethreaded=1" } else { "" }
+        "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1 $Include_freethreaded"
     }
 }
 
@@ -81,6 +84,7 @@ $PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version
 $PythonArchPath = Join-Path -Path $PythonVersionPath -ChildPath $Architecture
 
 $IsMSI = $PythonExecName -match "msi"
+$IsFreeThreaded = $Architecture -match "-freethreaded"
 
 $MajorVersion = $Version.Split('.')[0]
 $MinorVersion = $Version.Split('.')[1]
@@ -120,21 +124,36 @@ Write-Host "Copy Python binaries to $PythonArchPath"
 Copy-Item -Path ./$PythonExecName -Destination $PythonArchPath | Out-Null
 
 Write-Host "Install Python $Version in $PythonToolcachePath..."
-$ExecParams = Get-ExecParams -IsMSI $IsMSI -PythonArchPath $PythonArchPath
+$ExecParams = Get-ExecParams -IsMSI $IsMSI -IsFreeThreaded $IsFreeThreaded -PythonArchPath $PythonArchPath
 
 cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet"
 if ($LASTEXITCODE -ne 0) {
     Throw "Error happened during Python installation"
 }
 
+# print out all files in $PythonArchPath
+Write-Host "Files in $PythonArchPath"
+$files = Get-ChildItem -Path $PythonArchPath -File -Recurse
+Write-Output $files
+
+if ($IsFreeThreaded) {
+    # Delete python.exe and create a symlink to free-threaded exe
+    Remove-Item -Path "$PythonArchPath\python.exe" -Force
+    New-Item -Path "$PythonArchPath\python.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python${MajorVersion}.${MinorVersion}t.exe"
+}
+
 Write-Host "Create `python3` symlink"
 if ($MajorVersion -ne "2") {
     New-Item -Path "$PythonArchPath\python3.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python.exe"
 }
 
 Write-Host "Install and upgrade Pip"
+$Env:PIP_ROOT_USER_ACTION = "ignore"
 $PythonExePath = Join-Path -Path $PythonArchPath -ChildPath "python.exe"
-cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip --no-warn-script-location --root-user-action=ignore"
+cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade --force-reinstall pip --no-warn-script-location"
+if ($LASTEXITCODE -ne 0) {
+    Throw "Error happened during pip installation / upgrade"
+}
 
 Write-Host "Create complete file"
 New-Item -ItemType File -Path $PythonVersionPath -Name "$Architecture.complete" | Out-Null

tests/python-tests.ps1

@@ -7,6 +7,8 @@ param (
     $Architecture
 )
 
+$HardwareArchitecture = $Architecture -replace "-freethreaded", ""
+
 Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
 Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1")
 Import-Module (Join-Path $PSScriptRoot "../builders/python-version.psm1")
@@ -58,7 +60,7 @@ Describe "Tests" {
     #     }
     # }
 
-    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($Architecture -ne "arm64"))) {
+    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($HardwareArchitecture -ne "arm64"))) {
         It "Check if sqlite3 module is installed" {
             "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
         }
@@ -103,4 +105,8 @@ Describe "Tests" {
     It "Check urlopen with HTTPS works" {
         "python ./sources/python-urlopen-https.py" | Should -ReturnZeroExitCode
     }
+
+    It "Check a single dist-info per distribution is present" {
+        "python ./sources/dist-info.py" | Should -ReturnZeroExitCode
+    }
 }

tests/sources/dist-info.py

@@ -0,0 +1,24 @@
+import glob
+import os.path
+import sysconfig
+from collections import defaultdict
+
+
+def check_dist_info():
+    paths = set([sysconfig.get_path("purelib"), sysconfig.get_path("platlib")])
+    versions = defaultdict(list)
+    for path in paths:
+        pattern = os.path.join(path, "*.dist-info")
+        for dist_info in glob.glob(pattern):
+            name = os.path.basename(dist_info).split("-", maxsplit=1)[0]
+            versions[name].append(dist_info)
+    exit_code = 0
+    for name in versions:
+        if len(versions[name]) > 1:
+            print("multiple dist-info found for {}: {}".format(name, versions[name]))
+            exit_code = 1
+    exit(exit_code)
+
+
+if __name__ == "__main__":
+    check_dist_info()

tests/sources/python-config-test.py

@@ -8,27 +8,33 @@ os_type = platform.system()
 version = sys.argv[1]
 nativeVersion = sys.argv[2]
 architecture = sys.argv[3]
+hw_architecture = architecture.replace('-freethreaded', '')
 
 versions=version.split(".")
 version_major=int(versions[0])
 version_minor=int(versions[1])
 
-pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (architecture == "arm64"))
+pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (hw_architecture == "arm64"))
 
 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
 
 is_shared = sysconfig.get_config_var('Py_ENABLE_SHARED')
 have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE")
+is_free_threaded = sysconfig.get_config_var('Py_GIL_DISABLED')
 
 ### Define expected variables
 if os_type == 'Linux': expected_ld_library_extension = 'so'
 if os_type == 'Darwin': expected_ld_library_extension = 'dylib'
+if is_free_threaded:
+    framework_name = 'PythonT.framework'
+else:
+    framework_name = 'Python.framework'
 
 if pkg_installer:
-    expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib'
+    expected_lib_dir_path = f'/Library/Frameworks/{framework_name}/Versions/{version_major}.{version_minor}/lib'
 else:
-    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/x64/lib'
+    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/{architecture}/lib'
 
 # Check modules
 ### Validate libraries path
@@ -54,15 +60,15 @@ else:
 if os_type == 'Darwin':
     ### Validate openssl links
     if version_major == 3 and version_minor < 7:
-        expected_ldflags = '-L/usr/local/opt/openssl@1.1/lib'
+        expected_ldflags = '-L/usr/local/opt/openssl@3/lib'
         ldflags = sysconfig.get_config_var('LDFLAGS')
 
         if not expected_ldflags in ldflags:
             print('Invalid ldflags: %s; Expected: %s' % (ldflags, expected_ldflags))
             exit(1)
     else:
-        expected_openssl_includes = '-I/usr/local/opt/openssl@1.1/include'
-        expected_openssl_ldflags ='-L/usr/local/opt/openssl@1.1/lib'
+        expected_openssl_includes = '-I/usr/local/opt/openssl@3/include'
+        expected_openssl_ldflags ='-L/usr/local/opt/openssl@3/lib'
 
         openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES')
         openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS')