Compare commits

..

4 Commits

Author SHA1 Message Date
aparnajyothi-y
d80ea3a25f Delete builders/nix-python-builder.psm1 2024-06-24 15:07:52 +05:30
aparnajyothi-y
7a302f1496 Update build-python-packages.yml 2024-06-24 15:03:36 +05:30
aparnajyothi-y
e18dd16151 Update win-python-builder.psm1 2024-06-20 14:46:12 +05:30
aparnajyothi-y
606312b02e Update build-python-packages.yml 2024-06-20 14:45:24 +05:30
18 changed files with 475 additions and 3343 deletions

View File

@@ -12,15 +12,10 @@ on:
required: true required: true
type: boolean type: boolean
default: false default: false
THREADING_BUILD_MODES:
description: 'CPython threading build modes'
required: true
type: string
default: 'default,freethreaded'
PLATFORMS: PLATFORMS:
description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)' description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
required: true required: true
default: 'ubuntu-20.04,ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,ubuntu-24.04_arm64,macos-13_x64,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-2019_arm64' default: 'ubuntu-20.04,ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,macos-11_x64,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-2019_arm64'
pull_request: pull_request:
paths-ignore: paths-ignore:
- 'versions-manifest.json' - 'versions-manifest.json'
@@ -44,43 +39,31 @@ jobs:
- name: Generate execution matrix - name: Generate execution matrix
id: generate-matrix id: generate-matrix
run: | run: |
[String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,ubuntu-24.04_arm64,macos-13,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-2019_arm64' }}".Split(",").Trim() [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,macos-11,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-2019_arm64' }}".Split(",").Trim()
[String[]]$buildModes = "${{ inputs.threading_build_modes || 'default' }}".Split(",").Trim()
$matrix = @() $matrix = @()
foreach ($configuration in $configurations) { foreach ($configuration in $configurations) {
foreach ($buildMode in $buildModes) { $parts = $configuration.Split("_")
$parts = $configuration.Split("_") $os = $parts[0]
$os = $parts[0] $arch = if ($parts[1]) {$parts[1]} else {"x64"}
$arch = if ($parts[1]) {$parts[1]} else {"x64"}
switch -wildcard ($os) {
"*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
"*macos*" { $platform = 'darwin' }
"*windows*" { $platform = 'win32' }
}
if ($configuration -eq "ubuntu-22.04_arm64") { switch -wildcard ($os) {
$os = "setup-actions-ubuntu-arm64-2-core" "*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
} "*macos*" { $platform = 'darwin' }
elseif ($configuration -eq "ubuntu-24.04_arm64") { "*windows*" { $platform = 'win32' }
$os = "setup-actions-ubuntu24-arm64-2-core" }
}
elseif ($configuration -eq "windows-2019_arm64") { if ($configuration -eq "ubuntu-22.04_arm64") {
$os = "setup-actions-windows-arm64-4-core" $os = "setup-actions-ubuntu-arm64-2-core"
} }elseif ($configuration -eq "windows-2019_arm64") {
$os = "setup-actions-windows-arm64-4-core"
}
$matrix += @{
'platform' = $platform
'os' = $os
'arch' = $arch
if ($buildMode -eq "freethreaded") {
if ([semver]"${{ inputs.VERSION }}" -lt [semver]"3.13.0") {
continue;
}
$arch += "-freethreaded"
}
$matrix += @{
'platform' = $platform
'os' = $os
'arch' = $arch
}
} }
} }
echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT
@@ -95,44 +78,39 @@ jobs:
env: env:
ARTIFACT_NAME: python-${{ inputs.VERSION || '3.12.3' }}-${{ matrix.platform }}-${{ matrix.arch }} ARTIFACT_NAME: python-${{ inputs.VERSION || '3.12.3' }}-${{ matrix.platform }}-${{ matrix.arch }}
steps: steps:
- name: Setup Environment on Windows ARM64 Runner - name: Setup Environment on Windows ARM64 Runner
if: matrix.os == 'setup-actions-windows-arm64-4-core' if: matrix.os == 'setup-actions-windows-arm64-4-core'
shell: powershell shell: powershell
run: | run: |
# Install Chocolatey # Install Chocolatey
Set-ExecutionPolicy Bypass -Scope Process -Force Set-ExecutionPolicy Bypass -Scope Process -Force
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072 [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
echo "C:\ProgramData\Chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8 echo "C:\ProgramData\Chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
# Install PowerShell # Install PowerShell
choco install powershell-core -y choco install powershell-core -y
echo "C:\Program Files\PowerShell\7" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8 echo "C:\Program Files\PowerShell\7" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
# Install Git # Install Git
choco install git -y choco install git -y
echo "C:\Program Files\Git\cmd" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8 echo "C:\Program Files\Git\cmd" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
# Install 7-Zip
choco install 7zip -y
echo "C:\ProgramData\chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
# Install 7-Zip
choco install 7zip -y
echo "C:\ProgramData\chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
submodules: true submodules: true
- name: Adding Symlink
if: matrix.os == 'macos-13'
run: ln -sf /usr/local/opt/tcl-tk@8 /usr/local/opt/tcl-tk
- name: Build Python ${{ env.VERSION }} - name: Build Python ${{ env.VERSION }}
run: | run: |
./builders/build-python.ps1 -Version $env:VERSION ` ./builders/build-python.ps1 -Version $env:VERSION `
-Platform ${{ matrix.platform }} -Architecture ${{ matrix.arch }} -Platform ${{ matrix.platform }} -Architecture ${{ matrix.arch }}
- name: Publish artifact - name: Publish artifact
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
@@ -150,28 +128,29 @@ jobs:
env: env:
ARTIFACT_NAME: python-${{ inputs.VERSION || '3.12.3' }}-${{ matrix.platform }}-${{ matrix.arch }} ARTIFACT_NAME: python-${{ inputs.VERSION || '3.12.3' }}-${{ matrix.platform }}-${{ matrix.arch }}
steps: steps:
- name: Setup Environment on Windows ARM64 Runner - name: Setup Environment on Windows ARM64 Runner
if: matrix.os == 'setup-actions-windows-arm64-4-core' if: matrix.os == 'setup-actions-windows-arm64-4-core'
shell: powershell shell: powershell
run: | run: |
# Install Chocolatey # Install Chocolatey
Set-ExecutionPolicy Bypass -Scope Process -Force Set-ExecutionPolicy Bypass -Scope Process -Force
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072 [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
echo "C:\ProgramData\Chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8 echo "C:\ProgramData\Chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
# Install PowerShell # Install PowerShell
choco install powershell-core -y choco install powershell-core -y
echo "C:\Program Files\PowerShell\7" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8 echo "C:\Program Files\PowerShell\7" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
# Install Git # Install Git
choco install git -y choco install git -y
echo "C:\Program Files\Git\cmd" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8 echo "C:\Program Files\Git\cmd" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
# Install 7-Zip # Install 7-Zip
choco install 7zip -y choco install 7zip -y
echo "C:\ProgramData\chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8 echo "C:\ProgramData\chocolatey\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -192,6 +171,7 @@ jobs:
name: ${{ env.ARTIFACT_NAME }} name: ${{ env.ARTIFACT_NAME }}
path: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }} path: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
- name: Extract files - name: Extract files
run: | run: |
if ('${{ matrix.platform }}' -eq 'win32') { if ('${{ matrix.platform }}' -eq 'win32') {
@@ -220,9 +200,6 @@ jobs:
python-version: ${{ env.VERSION }} python-version: ${{ env.VERSION }}
architecture: ${{ matrix.arch }} architecture: ${{ matrix.arch }}
- name: Python version
run: python -VVV
- name: Verbose sysconfig dump - name: Verbose sysconfig dump
if: runner.os == 'Linux' || runner.os == 'macOS' if: runner.os == 'Linux' || runner.os == 'macOS'
run: python ./sources/python-config-output.py run: python ./sources/python-config-output.py
@@ -233,7 +210,6 @@ jobs:
run: | run: |
$pythonLocation = which python $pythonLocation = which python
if ('${{ matrix.platform }}' -eq 'darwin') { otool -L $pythonLocation } else { ldd $pythonLocation } if ('${{ matrix.platform }}' -eq 'darwin') { otool -L $pythonLocation } else { ldd $pythonLocation }
- name: Run tests - name: Run tests
run: | run: |
Install-Module Pester -Force -Scope CurrentUser -SkipPublisherCheck Install-Module Pester -Force -Scope CurrentUser -SkipPublisherCheck
@@ -268,7 +244,6 @@ jobs:
release_name: ${{ env.VERSION }} release_name: ${{ env.VERSION }}
body: | body: |
Python ${{ env.VERSION }} Python ${{ env.VERSION }}
- name: Generate hash for packages - name: Generate hash for packages
run: | run: |
$childItems = Get-Childitem -Path '.' $childItems = Get-Childitem -Path '.'

View File

@@ -13,5 +13,4 @@ jobs:
name: CodeQL analysis name: CodeQL analysis
uses: actions/reusable-workflows/.github/workflows/codeql-analysis.yml@main uses: actions/reusable-workflows/.github/workflows/codeql-analysis.yml@main
with: with:
languages: '["python"]' languages: "['python']"

View File

@@ -14,8 +14,8 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04, macos-13, ubuntu-latest] os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04]
python: [3.9.13, 3.10.11, 3.11.8, 3.12.7, 3.13.0] python: [3.9.13, 3.10.11, 3.11.8]
steps: steps:
- name: setup-python ${{ matrix.python }} - name: setup-python ${{ matrix.python }}
uses: actions/setup-python@v5 uses: actions/setup-python@v5

View File

@@ -17,7 +17,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04, ubuntu-latest, macos-13] os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04]
steps: steps:
- name: Setup Python ${{ github.event.inputs.version }} - name: Setup Python ${{ github.event.inputs.version }}
uses: actions/setup-python@main uses: actions/setup-python@main

View File

@@ -68,10 +68,10 @@ class macOSPythonBuilder : NixPythonBuilder {
### and then add the appropriate paths for the header and library files to configure command. ### and then add the appropriate paths for the header and library files to configure command.
### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies) ### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies)
if ($this.Version -lt "3.7.0") { if ($this.Version -lt "3.7.0") {
$env:LDFLAGS = "-L/usr/local/opt/openssl@3/lib -L/usr/local/opt/zlib/lib" $env:LDFLAGS = "-L/usr/local/opt/openssl@1.1/lib -L/usr/local/opt/zlib/lib"
$env:CFLAGS = "-I/usr/local/opt/openssl@3/include -I/usr/local/opt/zlib/include" $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/zlib/include"
} else { } else {
$configureString += " --with-openssl=/usr/local/opt/openssl@3" $configureString += " --with-openssl=/usr/local/opt/openssl@1.1"
# For Python 3.7.2 and 3.7.3 we need to provide PATH for zlib to pack it properly. Otherwise the build will fail # For Python 3.7.2 and 3.7.3 we need to provide PATH for zlib to pack it properly. Otherwise the build will fail
# with the error: zipimport.ZipImportError: can't decompress data; zlib not available # with the error: zipimport.ZipImportError: can't decompress data; zlib not available
@@ -80,9 +80,9 @@ class macOSPythonBuilder : NixPythonBuilder {
$env:CFLAGS = "-I/usr/local/opt/zlib/include" $env:CFLAGS = "-I/usr/local/opt/zlib/include"
} }
# if ($this.Version -gt "3.7.12") { if ($this.Version -gt "3.7.12") {
$configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include/tcl-tk' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'" $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'"
# } }
if ($this.Version -eq "3.7.17") { if ($this.Version -eq "3.7.17") {
$env:LDFLAGS += " -L$(brew --prefix bzip2)/lib -L$(brew --prefix readline)/lib -L$(brew --prefix ncurses)/lib" $env:LDFLAGS += " -L$(brew --prefix bzip2)/lib -L$(brew --prefix readline)/lib -L$(brew --prefix ncurses)/lib"
@@ -151,37 +151,6 @@ class macOSPythonBuilder : NixPythonBuilder {
return $pkgLocation return $pkgLocation
} }
[string] GetFrameworkName() {
<#
.SYNOPSIS
Get the Python installation Package name.
#>
if ($this.IsFreeThreaded()) {
return "PythonT.framework"
} else {
return "Python.framework"
}
}
[string] GetPkgChoices() {
<#
.SYNOPSIS
Reads the configuration XML file for the Python installer
#>
$config = if ($this.IsFreeThreaded()) { "freethreaded" } else { "default" }
$choicesFile = Join-Path $PSScriptRoot "../config/macos-pkg-choices-$($config).xml"
$choicesTemplate = Get-Content -Path $choicesFile -Raw
$variablesToReplace = @{
"{{__VERSION_MAJOR_MINOR__}}" = "$($this.Version.Major).$($this.Version.Minor)";
}
$variablesToReplace.keys | ForEach-Object { $choicesTemplate = $choicesTemplate.Replace($_, $variablesToReplace[$_]) }
return $choicesTemplate
}
[void] CreateInstallationScriptPkg() { [void] CreateInstallationScriptPkg() {
<# <#
.SYNOPSIS .SYNOPSIS
@@ -196,8 +165,6 @@ class macOSPythonBuilder : NixPythonBuilder {
"{{__VERSION_FULL__}}" = $this.Version; "{{__VERSION_FULL__}}" = $this.Version;
"{{__PKG_NAME__}}" = $this.GetPkgName(); "{{__PKG_NAME__}}" = $this.GetPkgName();
"{{__ARCH__}}" = $this.Architecture; "{{__ARCH__}}" = $this.Architecture;
"{{__FRAMEWORK_NAME__}}" = $this.GetFrameworkName();
"{{__PKG_CHOICES__}}" = $this.GetPkgChoices();
} }
$variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) } $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }

View File

@@ -1,169 +0,0 @@
using module "./python-builder.psm1"
class NixPythonBuilder : PythonBuilder {
<#
.SYNOPSIS
Base Python builder class for *Nix systems.
.DESCRIPTION
Contains methods that required to build Python artifact for *nix systems. Inherited from base PythonBuilder class.
.PARAMETER version
The version of Python that should be built.
.PARAMETER Platform
The type of platform for which Python should be built.
.PARAMETER PlatformVersion
The version of platform for which Python should be built.
.PARAMETER InstallationTemplateName
The name of template that will be used to create installation script for generated Python artifact.
.PARAMETER InstallationScriptName
The name of installation script that will be generated for Python artifact.
.PARAMETER OutputArtifactName
The name of archive with Python binaries that will be generated as part of Python artifact.
#>
[string] $InstallationTemplateName
[string] $InstallationScriptName
[string] $OutputArtifactName
NixPythonBuilder(
[semver] $version,
[string] $architecture,
[string] $platform
) : Base($version, $architecture, $platform) {
$this.InstallationTemplateName = "nix-setup-template.sh"
$this.InstallationScriptName = "setup.sh"
$this.OutputArtifactName = "python-$Version-$Platform-$Architecture.tar.gz"
}
[uri] GetSourceUri() {
<#
.SYNOPSIS
Get base Python URI and return complete URI for Python sources.
#>
$base = $this.GetBaseUri()
$versionName = $this.GetBaseVersion()
$nativeVersion = Convert-Version -version $this.Version
return "${base}/${versionName}/Python-${nativeVersion}.tgz"
}
[string] GetPythonBinary() {
<#
.SYNOPSIS
Return name of Python binary.
#>
if ($this.Version.Major -eq 2) { $pythonBinary = "python" } else { $pythonBinary = "python3" }
return $pythonBinary
}
[string] Download() {
<#
.SYNOPSIS
Download Python sources and extract them at temporary work folder. Returns expanded archive location path.
#>
$sourceUri = $this.GetSourceUri()
Write-Host "Sources URI: $sourceUri"
$archiveFilepath = Download-File -Uri $sourceUri -OutputFolder $this.WorkFolderLocation
$expandedSourceLocation = Join-Path -Path $this.TempFolderLocation -ChildPath "SourceCode"
New-Item -Path $expandedSourceLocation -ItemType Directory
Extract-TarArchive -ArchivePath $archiveFilepath -OutputDirectory $expandedSourceLocation
Write-Debug "Done; Sources location: $expandedSourceLocation"
return $expandedSourceLocation
}
[void] CreateInstallationScript() {
<#
.SYNOPSIS
Create Python artifact installation script based on template specified in InstallationTemplateName property.
#>
$installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName -ItemType File
$installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath $this.InstallationTemplateName
$installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
$variablesToReplace = @{
"{{__VERSION_FULL__}}" = $this.Version;
"{{__ARCH__}}" = $this.Architecture;
}
$variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
$installationTemplateContent | Out-File -FilePath $installationScriptLocation
Write-Debug "Done; Installation script location: $installationScriptLocation)"
}
[void] Make() {
<#
.SYNOPSIS
Executes "make" and "make install" commands for configured build sources. Make output will be writen in build_output.txt located in artifact location folder.
#>
Write-Debug "make Python $($this.Version)-$($this.Architecture) $($this.Platform)"
$buildOutputLocation = New-Item -Path $this.WorkFolderLocation -Name "build_output.txt" -ItemType File
Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue
Execute-Command -Command "make install" -ErrorAction Continue
Write-Debug "Done; Make log location: $buildOutputLocation"
}
[void] CopyBuildResults() {
$buildFolder = $this.GetFullPythonToolcacheLocation()
Move-Item -Path "$buildFolder/*" -Destination $this.WorkFolderLocation
}
[void] ArchiveArtifact() {
$OutputPath = Join-Path $this.ArtifactFolderLocation $this.OutputArtifactName
Create-TarArchive -SourceFolder $this.WorkFolderLocation -ArchivePath $OutputPath
}
[void] Build() {
<#
.SYNOPSIS
Build Python artifact from sources.
#>
Write-Host "Prepare Python Hostedtoolcache location..."
$this.PreparePythonToolcacheLocation()
Write-Host "Prepare system environment..."
$this.PrepareEnvironment()
Write-Host "Download Python $($this.Version)[$($this.Architecture)] sources..."
$sourcesLocation = $this.Download()
Push-Location -Path $sourcesLocation
Write-Host "Configure for $($this.Platform)..."
$this.Configure()
Write-Host "Make for $($this.Platform)..."
$this.Make()
Pop-Location
Write-Host "Generate structure dump"
New-ToolStructureDump -ToolPath $this.GetFullPythonToolcacheLocation() -OutputFolder $this.WorkFolderLocation
Write-Host "Copying build results to destination location"
$this.CopyBuildResults()
Write-Host "Create installation script..."
$this.CreateInstallationScript()
Write-Host "Archive artifact..."
$this.ArchiveArtifact()
}
}

View File

@@ -94,24 +94,6 @@ class PythonBuilder {
return "$($this.Version.Major).$($this.Version.Minor).$($this.Version.Patch)" return "$($this.Version.Major).$($this.Version.Minor).$($this.Version.Patch)"
} }
[string] GetHardwareArchitecture() {
<#
.SYNOPSIS
The hardware architecture (x64, arm64) without any Python free threading suffix.
#>
return $this.Architecture.Replace("-freethreaded", "")
}
[bool] IsFreeThreaded() {
<#
.SYNOPSIS
Check if Python version is free threaded.
#>
return $this.Architecture.EndsWith("-freethreaded")
}
[void] PreparePythonToolcacheLocation() { [void] PreparePythonToolcacheLocation() {
<# <#
.SYNOPSIS .SYNOPSIS

View File

@@ -37,14 +37,6 @@ class UbuntuPythonBuilder : NixPythonBuilder {
$configureString += " --enable-shared" $configureString += " --enable-shared"
$configureString += " --enable-optimizations" $configureString += " --enable-optimizations"
if ($this.IsFreeThreaded()) {
if ($this.Version -lt "3.13.0") {
Write-Host "Python versions lower than 3.13.0 do not support free threading"
exit 1
}
$configureString += " --disable-gil"
}
### Compile with support of loadable sqlite extensions. ### Compile with support of loadable sqlite extensions.
### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension) ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
$configureString += " --enable-loadable-sqlite-extensions" $configureString += " --enable-loadable-sqlite-extensions"

View File

@@ -1,3 +1,4 @@
using module "./python-builder.psm1" using module "./python-builder.psm1"
class WinPythonBuilder : PythonBuilder { class WinPythonBuilder : PythonBuilder {
@@ -54,16 +55,17 @@ class WinPythonBuilder : PythonBuilder {
#> #>
$ArchitectureExtension = "" $ArchitectureExtension = ""
if ($this.GetHardwareArchitecture() -eq "x64") { if ($this.Architecture -eq "x64") {
if ($this.Version -ge "3.5") { if ($this.Version -ge "3.5") {
$ArchitectureExtension = "-amd64" $ArchitectureExtension = "-amd64"
} else { } else {
$ArchitectureExtension = ".amd64" $ArchitectureExtension = ".amd64"
} }
} elseif ($this.GetHardwareArchitecture() -eq "arm64") { }elseif ($this.Architecture -eq "arm64") {
$ArchitectureExtension = "-arm64" $ArchitectureExtension = "-arm64"
} }
return $ArchitectureExtension return $ArchitectureExtension
} }
@@ -113,7 +115,6 @@ class WinPythonBuilder : PythonBuilder {
$variablesToReplace = @{ $variablesToReplace = @{
"{{__ARCHITECTURE__}}" = $this.Architecture; "{{__ARCHITECTURE__}}" = $this.Architecture;
"{{__HARDWARE_ARCHITECTURE__}}" = $this.GetHardwareArchitecture();
"{{__VERSION__}}" = $this.Version; "{{__VERSION__}}" = $this.Version;
"{{__PYTHON_EXEC_NAME__}}" = $pythonExecName "{{__PYTHON_EXEC_NAME__}}" = $pythonExecName
} }

View File

@@ -1,8 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<array>
<dict>
</dict>
</array>
</plist>

View File

@@ -1,14 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<array>
<dict>
<key>attributeSetting</key>
<integer>1</integer>
<key>choiceAttribute</key>
<string>selected</string>
<key>choiceIdentifier</key>
<string>org.python.Python.PythonTFramework-{{__VERSION_MAJOR_MINOR__}}</string>
</dict>
</array>
</plist>

View File

@@ -1,5 +1,5 @@
{ {
"regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+(-freethreaded)?)", "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+)",
"groups": { "groups": {
"arch": 4, "arch": 4,
"platform": 2, "platform": 2,

View File

@@ -2,11 +2,6 @@ set -e
PYTHON_FULL_VERSION="{{__VERSION_FULL__}}" PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
PYTHON_PKG_NAME="{{__PKG_NAME__}}" PYTHON_PKG_NAME="{{__PKG_NAME__}}"
PYTHON_FRAMEWORK_NAME="{{__FRAMEWORK_NAME__}}"
PYTHON_PKG_CHOICES=$(cat << 'EOF'
{{__PKG_CHOICES__}}
EOF
)
ARCH="{{__ARCH__}}" ARCH="{{__ARCH__}}"
MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1) MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2) MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
@@ -25,7 +20,7 @@ fi
PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
PYTHON_FRAMEWORK_PATH="/Library/Frameworks/${PYTHON_FRAMEWORK_NAME}/Versions/${MAJOR_VERSION}.${MINOR_VERSION}" PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}" PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
echo "Check if Python hostedtoolcache folder exist..." echo "Check if Python hostedtoolcache folder exist..."
@@ -43,11 +38,8 @@ else
done done
fi fi
PYTHON_PKG_CHOICES_FILES=$(mktemp)
echo "$PYTHON_PKG_CHOICES" > $PYTHON_PKG_CHOICES_FILES
echo "Install Python binaries from prebuilt package" echo "Install Python binaries from prebuilt package"
sudo installer -pkg $PYTHON_PKG_NAME -applyChoiceChangesXML $PYTHON_PKG_CHOICES_FILES -target / sudo installer -pkg $PYTHON_PKG_NAME -target /
echo "Create hostedtoolcach symlinks (Required for the backward compatibility)" echo "Create hostedtoolcach symlinks (Required for the backward compatibility)"
echo "Create Python $PYTHON_FULL_VERSION folder" echo "Create Python $PYTHON_FULL_VERSION folder"
@@ -61,9 +53,7 @@ ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib
echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)" echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
chmod +x python
# Note that bin is a symlink so referencing .. from bin will not work as expected
cd bin/ cd bin/
# This symlink already exists if Python version with the same major.minor version is installed, # This symlink already exists if Python version with the same major.minor version is installed,
@@ -72,15 +62,11 @@ if [ ! -f $PYTHON_MAJOR_MINOR ]; then
ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
fi fi
if [ ! -f $PYTHON_MAJOR ]; then
ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR
fi
if [ ! -f python ]; then if [ ! -f python ]; then
ln -s $PYTHON_MAJOR_DOT_MINOR python ln -s $PYTHON_MAJOR_DOT_MINOR python
fi fi
chmod +x $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
echo "Upgrading pip..." echo "Upgrading pip..."
export PIP_ROOT_USER_ACTION=ignore export PIP_ROOT_USER_ACTION=ignore

View File

@@ -24,9 +24,9 @@ echo "Check if Python hostedtoolcache folder exist..."
if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
echo "Creating Python hostedtoolcache folder..." echo "Creating Python hostedtoolcache folder..."
mkdir -p $PYTHON_TOOLCACHE_PATH mkdir -p $PYTHON_TOOLCACHE_PATH
elif [ -d $PYTHON_TOOLCACHE_VERSION_ARCH_PATH ]; then elif [ -d $PYTHON_TOOLCACHE_VERSION_PATH ]; then
echo "Deleting Python $PYTHON_FULL_VERSION ($ARCH)" echo "Deleting Python $PYTHON_FULL_VERSION"
rm -rf $PYTHON_TOOLCACHE_VERSION_ARCH_PATH rm -rf $PYTHON_TOOLCACHE_VERSION_PATH
fi fi
echo "Create Python $PYTHON_FULL_VERSION folder" echo "Create Python $PYTHON_FULL_VERSION folder"

View File

@@ -1,5 +1,4 @@
[String] $Architecture = "{{__ARCHITECTURE__}}" [String] $Architecture = "{{__ARCHITECTURE__}}"
[String] $HardwareArchitecture = "{{__HARDWARE_ARCHITECTURE__}}"
[String] $Version = "{{__VERSION__}}" [String] $Version = "{{__VERSION__}}"
[String] $PythonExecName = "{{__PYTHON_EXEC_NAME__}}" [String] $PythonExecName = "{{__PYTHON_EXEC_NAME__}}"
@@ -26,7 +25,7 @@ function Remove-RegistryEntries {
[Parameter(Mandatory)][Int32] $MinorVersion [Parameter(Mandatory)][Int32] $MinorVersion
) )
$versionFilter = Get-RegistryVersionFilter -Architecture $HardwareArchitecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion $versionFilter = Get-RegistryVersionFilter -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
$regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products" $regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products"
if (Test-Path -Path Registry::$regPath) { if (Test-Path -Path Registry::$regPath) {
@@ -62,15 +61,13 @@ function Remove-RegistryEntries {
function Get-ExecParams { function Get-ExecParams {
param( param(
[Parameter(Mandatory)][Boolean] $IsMSI, [Parameter(Mandatory)][Boolean] $IsMSI,
[Parameter(Mandatory)][Boolean] $IsFreeThreaded,
[Parameter(Mandatory)][String] $PythonArchPath [Parameter(Mandatory)][String] $PythonArchPath
) )
if ($IsMSI) { if ($IsMSI) {
"TARGETDIR=$PythonArchPath ALLUSERS=1" "TARGETDIR=$PythonArchPath ALLUSERS=1"
} else { } else {
$Include_freethreaded = if ($IsFreeThreaded) { "Include_freethreaded=1" } else { "" } "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1"
"DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1 $Include_freethreaded"
} }
} }
@@ -84,7 +81,6 @@ $PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version
$PythonArchPath = Join-Path -Path $PythonVersionPath -ChildPath $Architecture $PythonArchPath = Join-Path -Path $PythonVersionPath -ChildPath $Architecture
$IsMSI = $PythonExecName -match "msi" $IsMSI = $PythonExecName -match "msi"
$IsFreeThreaded = $Architecture -match "-freethreaded"
$MajorVersion = $Version.Split('.')[0] $MajorVersion = $Version.Split('.')[0]
$MinorVersion = $Version.Split('.')[1] $MinorVersion = $Version.Split('.')[1]
@@ -124,24 +120,13 @@ Write-Host "Copy Python binaries to $PythonArchPath"
Copy-Item -Path ./$PythonExecName -Destination $PythonArchPath | Out-Null Copy-Item -Path ./$PythonExecName -Destination $PythonArchPath | Out-Null
Write-Host "Install Python $Version in $PythonToolcachePath..." Write-Host "Install Python $Version in $PythonToolcachePath..."
$ExecParams = Get-ExecParams -IsMSI $IsMSI -IsFreeThreaded $IsFreeThreaded -PythonArchPath $PythonArchPath $ExecParams = Get-ExecParams -IsMSI $IsMSI -PythonArchPath $PythonArchPath
cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet" cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet"
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Throw "Error happened during Python installation" Throw "Error happened during Python installation"
} }
# print out all files in $PythonArchPath
Write-Host "Files in $PythonArchPath"
$files = Get-ChildItem -Path $PythonArchPath -File -Recurse
Write-Output $files
if ($IsFreeThreaded) {
# Delete python.exe and create a symlink to free-threaded exe
Remove-Item -Path "$PythonArchPath\python.exe" -Force
New-Item -Path "$PythonArchPath\python.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python${MajorVersion}.${MinorVersion}t.exe"
}
Write-Host "Create `python3` symlink" Write-Host "Create `python3` symlink"
if ($MajorVersion -ne "2") { if ($MajorVersion -ne "2") {
New-Item -Path "$PythonArchPath\python3.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python.exe" New-Item -Path "$PythonArchPath\python3.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python.exe"

View File

@@ -7,8 +7,6 @@ param (
$Architecture $Architecture
) )
$HardwareArchitecture = $Architecture -replace "-freethreaded", ""
Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1") Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1") Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1")
Import-Module (Join-Path $PSScriptRoot "../builders/python-version.psm1") Import-Module (Join-Path $PSScriptRoot "../builders/python-version.psm1")
@@ -60,7 +58,7 @@ Describe "Tests" {
# } # }
# } # }
if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($HardwareArchitecture -ne "arm64"))) { if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($Architecture -ne "arm64"))) {
It "Check if sqlite3 module is installed" { It "Check if sqlite3 module is installed" {
"python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
} }

View File

@@ -8,31 +8,25 @@ os_type = platform.system()
version = sys.argv[1] version = sys.argv[1]
nativeVersion = sys.argv[2] nativeVersion = sys.argv[2]
architecture = sys.argv[3] architecture = sys.argv[3]
hw_architecture = architecture.replace('-freethreaded', '')
versions=version.split(".") versions=version.split(".")
version_major=int(versions[0]) version_major=int(versions[0])
version_minor=int(versions[1]) version_minor=int(versions[1])
pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (hw_architecture == "arm64")) pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (architecture == "arm64"))
lib_dir_path = sysconfig.get_config_var('LIBDIR') lib_dir_path = sysconfig.get_config_var('LIBDIR')
ld_library_name = sysconfig.get_config_var('LDLIBRARY') ld_library_name = sysconfig.get_config_var('LDLIBRARY')
is_shared = sysconfig.get_config_var('Py_ENABLE_SHARED') is_shared = sysconfig.get_config_var('Py_ENABLE_SHARED')
have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE") have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE")
is_free_threaded = sysconfig.get_config_var('Py_GIL_DISABLED')
### Define expected variables ### Define expected variables
if os_type == 'Linux': expected_ld_library_extension = 'so' if os_type == 'Linux': expected_ld_library_extension = 'so'
if os_type == 'Darwin': expected_ld_library_extension = 'dylib' if os_type == 'Darwin': expected_ld_library_extension = 'dylib'
if is_free_threaded:
framework_name = 'PythonT.framework'
else:
framework_name = 'Python.framework'
if pkg_installer: if pkg_installer:
expected_lib_dir_path = f'/Library/Frameworks/{framework_name}/Versions/{version_major}.{version_minor}/lib' expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib'
else: else:
expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/{architecture}/lib' expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/{architecture}/lib'
@@ -60,15 +54,15 @@ else:
if os_type == 'Darwin': if os_type == 'Darwin':
### Validate openssl links ### Validate openssl links
if version_major == 3 and version_minor < 7: if version_major == 3 and version_minor < 7:
expected_ldflags = '-L/usr/local/opt/openssl@3/lib' expected_ldflags = '-L/usr/local/opt/openssl@1.1/lib'
ldflags = sysconfig.get_config_var('LDFLAGS') ldflags = sysconfig.get_config_var('LDFLAGS')
if not expected_ldflags in ldflags: if not expected_ldflags in ldflags:
print('Invalid ldflags: %s; Expected: %s' % (ldflags, expected_ldflags)) print('Invalid ldflags: %s; Expected: %s' % (ldflags, expected_ldflags))
exit(1) exit(1)
else: else:
expected_openssl_includes = '-I/usr/local/opt/openssl@3/include' expected_openssl_includes = '-I/usr/local/opt/openssl@1.1/include'
expected_openssl_ldflags ='-L/usr/local/opt/openssl@3/lib' expected_openssl_ldflags ='-L/usr/local/opt/openssl@1.1/lib'
openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES') openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES')
openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS') openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS')

File diff suppressed because it is too large Load Diff