mplement script to build and prepare packages for Python

This commit is contained in:
Maxim Lobanov
2020-04-29 10:57:27 +03:00
commit c641695f6a
37 changed files with 2558 additions and 0 deletions

75
builders/build-python.ps1 Normal file
View File

@@ -0,0 +1,75 @@
using module "./builders/win-python-builder.psm1"
using module "./builders/ubuntu-python-builder.psm1"
using module "./builders/macos-python-builder.psm1"
<#
.SYNOPSIS
Generate Python artifact.
.DESCRIPTION
Main script that creates instance of PythonBuilder and builds of Python using specified parameters.
.PARAMETER Version
Required parameter. The version with which Python will be built.
.PARAMETER Architecture
Optional parameter. The architecture with which Python will be built. Using x64 by default.
.PARAMETER Platform
Required parameter. The platform for which Python will be built.
#>
param(
[Parameter (Mandatory=$true)][Version] $Version,
[Parameter (Mandatory=$true)][string] $Platform,
[string] $Architecture = "x64"
)
Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "common-helpers.psm1") -DisableNameChecking
Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "nix-helpers.psm1") -DisableNameChecking
Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "win-helpers.psm1") -DisableNameChecking
function Get-PythonBuilder {
<#
.SYNOPSIS
Wrapper for class constructor to simplify importing PythonBuilder.
.DESCRIPTION
Create instance of PythonBuilder with specified parameters.
.PARAMETER Version
The version with which Python will be built.
.PARAMETER Architecture
The architecture with which Python will be built.
.PARAMETER Platform
The platform for which Python will be built.
#>
param (
[version] $Version,
[string] $Architecture,
[string] $Platform
)
$Platform = $Platform.ToLower()
if ($Platform -match 'windows') {
$builder = [WinPythonBuilder]::New($Version, $Architecture, $Platform)
} elseif ($Platform -match 'ubuntu') {
$builder = [UbuntuPythonBuilder]::New($Version, $Architecture, $Platform)
} elseif ($Platform -match 'macos') {
$builder = [macOSPythonBuilder]::New($Version, $Architecture, $Platform)
} else {
Write-Host "##vso[task.logissue type=error;] Invalid platform: $Platform"
exit 1
}
return $builder
}
### Create Python builder instance, and build artifact
$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture -Platform $Platform
$Builder.Build()

View File

@@ -0,0 +1,58 @@
using module "./builders/nix-python-builder.psm1"
class macOSPythonBuilder : NixPythonBuilder {
<#
.SYNOPSIS
MacOS Python builder class.
.DESCRIPTION
Contains methods that required to build macOS Python artifact from sources. Inherited from base NixPythonBuilder.
.PARAMETER platform
The full name of platform for which Python should be built.
.PARAMETER version
The version of Python that should be built.
#>
macOSPythonBuilder(
[version] $version,
[string] $architecture,
[string] $platform
) : Base($version, $architecture, $platform) { }
[void] Configure() {
<#
.SYNOPSIS
Execute configure script with required parameters.
#>
$pythonBinariesLocation = $this.GetFullPythonToolcacheLocation()
$configureString = "./configure --prefix=$pythonBinariesLocation --enable-optimizations --enable-shared --with-lto"
### OS X 10.11, Apple no longer provides header files for the deprecated system version of OpenSSL.
### Solution is to install these libraries from a third-party package manager,
### and then add the appropriate paths for the header and library files to configure command.
### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies)
if ($this.Version -lt "3.7.0") {
$env:LDFLAGS="-L$(brew --prefix openssl)/lib"
$env:CFLAGS="-I$(brew --prefix openssl)/include"
} else {
$configureString += " --with-openssl=/usr/local/opt/openssl"
}
Execute-Command -Command $configureString
}
[void] PrepareEnvironment() {
<#
.SYNOPSIS
Prepare system environment by installing dependencies and required packages.
#>
### reinstall header files to Avoid issue with X11 headers on Mojave
$pkgName = "/Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg"
Execute-Command -Command "sudo installer -pkg $pkgName -target /"
}
}

View File

@@ -0,0 +1,162 @@
using module "./builders/python-builder.psm1"
class NixPythonBuilder : PythonBuilder {
<#
.SYNOPSIS
Base Python builder class for *Nix systems.
.DESCRIPTION
Contains methods that required to build Python artifact for *nix systems. Inherited from base PythonBuilder class.
.PARAMETER version
The version of Python that should be built.
.PARAMETER Platform
The type of platform for which Python should be built.
.PARAMETER PlatformVersion
The version of platform for which Python should be built.
.PARAMETER InstallationTemplateName
The name of template that will be used to create installation script for generated Python artifact.
.PARAMETER InstallationScriptName
The name of installation script that will be generated for Python artifact.
.PARAMETER OutputArtifactName
The name of archive with Python binaries that will be generated as part of Python artifact.
#>
[string] $InstallationTemplateName
[string] $InstallationScriptName
[string] $OutputArtifactName
NixPythonBuilder(
[version] $version,
[string] $architecture,
[string] $platform
) : Base($version, $architecture, $platform) {
$this.InstallationTemplateName = "nix-setup-template.sh"
$this.InstallationScriptName = "setup.sh"
$this.OutputArtifactName = "python-$Version-$Platform-$Architecture.tar.gz"
}
[uri] GetSourceUri() {
<#
.SYNOPSIS
Get base Python URI and return complete URI for Python sources.
#>
$base = $this.GetBaseUri()
return "${base}/$($this.Version)/Python-$($this.Version).tgz"
}
[string] GetPythonBinary() {
<#
.SYNOPSIS
Return name of Python binary.
#>
if ($this.Version.Major -eq 2) { $pythonBinary = "python" } else { $pythonBinary = "python3" }
return $pythonBinary
}
[string] Download() {
<#
.SYNOPSIS
Download Python sources and extract them at temporary work folder. Returns expanded archive location path.
#>
$sourceUri = $this.GetSourceUri()
Write-Host "Sources URI: $sourceUri"
$archiveFilepath = Download-File -Uri $sourceUri -OutputFolder $this.WorkFolderLocation
$expandedSourceLocation = Join-Path -Path $this.TempFolderLocation -ChildPath "SourceCode"
New-Item -Path $expandedSourceLocation -ItemType Directory
Extract-TarArchive -ArchivePath $archiveFilepath -OutputDirectory $expandedSourceLocation
Write-Debug "Done; Sources location: $expandedSourceLocation"
return $expandedSourceLocation
}
[void] CreateInstallationScript() {
<#
.SYNOPSIS
Create Python artifact installation script based on template specified in InstallationTemplateName property.
#>
$installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName -ItemType File
$installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath $this.InstallationTemplateName
$installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
$installationTemplateContent = $installationTemplateContent -f $this.Version.Major, $this.Version.Minor, $this.Version.Build
$installationTemplateContent | Out-File -FilePath $installationScriptLocation
Write-Debug "Done; Installation script location: $installationScriptLocation)"
}
[void] Make() {
<#
.SYNOPSIS
Executes "make" and "make install" commands for configured build sources. Make output will be writen in build_output.txt located in artifact location folder.
#>
Write-Debug "make Python $($this.Version)-$($this.Architecture) $($this.Platform)"
$buildOutputLocation = New-Item -Path $this.WorkFolderLocation -Name "build_output.txt" -ItemType File
Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue
Execute-Command -Command "make install" -ErrorAction Continue
Write-Debug "Done; Make log location: $buildOutputLocation"
}
[void] CopyBuildResults() {
$buildFolder = $this.GetFullPythonToolcacheLocation()
Move-Item -Path "$buildFolder/*" -Destination $this.WorkFolderLocation
}
[void] ArchiveArtifact() {
$OutputPath = Join-Path $this.ArtifactFolderLocation $this.OutputArtifactName
Create-TarArchive -SourceFolder $this.WorkFolderLocation -ArchivePath $OutputPath
}
[void] Build() {
<#
.SYNOPSIS
Build Python artifact from sources.
#>
Write-Host "Prepare Python Hostedtoolcache location..."
$this.PreparePythonToolcacheLocation()
Write-Host "Prepare system environment..."
$this.PrepareEnvironment()
Write-Host "Download Python $($this.Version)[$($this.Architecture)] sources..."
$sourcesLocation = $this.Download()
Push-Location -Path $sourcesLocation
Write-Host "Configure for $($this.Platform)..."
$this.Configure()
Write-Host "Make for $($this.Platform)..."
$this.Make()
Pop-Location
Write-Host "Generate structure dump"
New-ToolStructureDump -ToolPath $this.GetFullPythonToolcacheLocation() -OutputFolder $this.WorkFolderLocation
Write-Host "Copying build results to destination location"
$this.CopyBuildResults()
Write-Host "Create installation script..."
$this.CreateInstallationScript()
Write-Host "Archive artifact..."
$this.ArchiveArtifact()
}
}

View File

@@ -0,0 +1,97 @@
class PythonBuilder {
<#
.SYNOPSIS
Base Python builder class.
.DESCRIPTION
Base Python builder class that contains general builder methods.
.PARAMETER Version
The version of Python that should be built.
.PARAMETER Architecture
The architecture with which Python should be built.
.PARAMETER HostedToolcacheLocation
The location of hostedtoolcache artifacts. Using system AGENT_TOOLSDIRECTORY variable value.
.PARAMETER TempFolderLocation
The location of temporary files that will be used during Python generation. Using system TEMP directory.
.PARAMETER WorkFolderLocation
The location of generated Python artifact. Using system environment BUILD_STAGINGDIRECTORY variable value.
.PARAMETER ArtifactFolderLocation
The location of generated Python artifact. Using system environment BUILD_BINARIESDIRECTORY variable value.
.PARAMETER InstallationTemplatesLocation
The location of installation script template. Using "installers" folder from current repository.
#>
[version] $Version
[string] $Architecture
[string] $Platform
[string] $HostedToolcacheLocation
[string] $TempFolderLocation
[string] $WorkFolderLocation
[string] $ArtifactFolderLocation
[string] $InstallationTemplatesLocation
PythonBuilder ([version] $version, [string] $architecture, [string] $platform) {
$this.Version = $version
$this.Architecture = $architecture
$this.Platform = $platform
$this.HostedToolcacheLocation = $env:AGENT_TOOLSDIRECTORY
$this.TempFolderLocation = $env:BUILD_SOURCESDIRECTORY
$this.WorkFolderLocation = $env:BUILD_BINARIESDIRECTORY
$this.ArtifactFolderLocation = $env:BUILD_STAGINGDIRECTORY
$this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers"
}
[uri] GetBaseUri() {
<#
.SYNOPSIS
Return base URI for Python build sources.
#>
return "https://www.python.org/ftp/python"
}
[string] GetPythonToolcacheLocation() {
<#
.SYNOPSIS
Return path to Python hostedtoolcache folder.
#>
return "$($this.HostedToolcacheLocation)/Python"
}
[string] GetFullPythonToolcacheLocation() {
<#
.SYNOPSIS
Return full path to hostedtoolcache Python folder.
#>
$pythonToolcacheLocation = $this.GetPythonToolcacheLocation()
return "$pythonToolcacheLocation/$($this.Version)/$($this.Architecture)"
}
[void] PreparePythonToolcacheLocation() {
<#
.SYNOPSIS
Prepare system hostedtoolcache folder for new Python version.
#>
$pythonBinariesLocation = $this.GetFullPythonToolcacheLocation()
if (Test-Path $pythonBinariesLocation) {
Write-Host "Purge $pythonBinariesLocation folder..."
Remove-Item $pythonBinariesLocation -Recurse -Force
} else {
Write-Host "Create $pythonBinariesLocation folder..."
New-Item -ItemType Directory -Path $pythonBinariesLocation
}
}
}

View File

@@ -0,0 +1,85 @@
using module "./builders/nix-python-builder.psm1"
class UbuntuPythonBuilder : NixPythonBuilder {
<#
.SYNOPSIS
Ubuntu Python builder class.
.DESCRIPTION
Contains methods that required to build Ubuntu Python artifact from sources. Inherited from base NixPythonBuilder.
.PARAMETER platform
The full name of platform for which Python should be built.
.PARAMETER version
The version of Python that should be built.
#>
UbuntuPythonBuilder(
[version] $version,
[string] $architecture,
[string] $platform
) : Base($version, $architecture, $platform) { }
[void] Configure() {
<#
.SYNOPSIS
Execute configure script with required parameters.
#>
$pythonBinariesLocation = $this.GetFullPythonToolcacheLocation()
### To build Python with SO we must pass full path to lib folder to the linker
$env:LDFLAGS="-Wl,--rpath=${pythonBinariesLocation}/lib"
$configureString = "./configure --prefix=$pythonBinariesLocation --enable-shared --enable-optimizations"
if ($this.Version -lt "3.0.0") {
### Compile with ucs4 for Python 2.x. On 3.x, ucs4 is enabled by default
$configureString += " --enable-unicode=ucs4"
}
Execute-Command -Command $configureString
}
[void] PrepareEnvironment() {
<#
.SYNOPSIS
Prepare system environment by installing dependencies and required packages.
#>
if (($this.Version -gt "3.0.0") -and ($this.Version -lt "3.5.3")) {
Write-Host "Python3 versions lower than 3.5.3 are not supported"
exit 1
}
### Compile with tkinter support
if ($this.Version -gt "3.0.0") {
$tkinterInstallString = "sudo apt-get install -y --allow-downgrades python3-tk tk-dev"
} else {
$tkinterInstallString = "sudo apt install -y python-tk tk-dev"
}
Execute-Command -Command $tkinterInstallString
### Install dependent packages
@(
"make",
"build-essential",
"libssl-dev",
"zlib1g-dev",
"libbz2-dev",
"libsqlite3-dev",
"libncursesw5-dev",
"libreadline-dev",
"libgdbm-dev"
) | ForEach-Object {
Execute-Command -Command "sudo apt install -y $_"
}
if ($this.Platform -ne "ubuntu-1604") {
### On Ubuntu-1804, libgdbm-compat-dev has older modules that are no longer in libgdbm-dev
Execute-Command -Command "sudo apt install -y libgdbm-compat-dev"
}
}
}

View File

@@ -0,0 +1,141 @@
using module "./builders/python-builder.psm1"
class WinPythonBuilder : PythonBuilder {
<#
.SYNOPSIS
Base Python builder class for Windows systems.
.DESCRIPTION
Contains methods required for build Windows Python artifact. Inherited from base PythonBuilder class.
.PARAMETER version
The version of Python that should be built.
.PARAMETER architecture
The architecture with which Python should be built.
.PARAMETER InstallationTemplateName
The name of installation script template that will be used in generated artifact.
.PARAMETER InstallationScriptName
The name of generated installation script.
#>
[string] $InstallationTemplateName
[string] $InstallationScriptName
[string] $OutputArtifactName
WinPythonBuilder(
[version] $version,
[string] $architecture,
[string] $platform
) : Base($version, $architecture, $platform) {
$this.InstallationTemplateName = "win-setup-template.ps1"
$this.InstallationScriptName = "setup.ps1"
$this.OutputArtifactName = "python-$Version-$Platform-$Architecture.zip"
}
[string] GetPythonExtension() {
<#
.SYNOPSIS
Return extension for required version of Python executable.
#>
$extension = if ($this.Version -lt "3.5" -and $this.Version -ge "2.5") { ".msi" } else { ".exe" }
return $extension
}
[string] GetArchitectureExtension() {
<#
.SYNOPSIS
Return architecture suffix for Python executable.
#>
$ArchitectureExtension = ""
if ($this.Architecture -eq "x64") {
if ($this.Version -ge "3.5") {
$ArchitectureExtension = "-amd64"
} else {
$ArchitectureExtension = ".amd64"
}
}
return $ArchitectureExtension
}
[uri] GetSourceUri() {
<#
.SYNOPSIS
Get base Python URI and return complete URI for Python installation executable.
#>
$base = $this.GetBaseUri()
$architecture = $this.GetArchitectureExtension()
$extension = $this.GetPythonExtension()
$uri = "${base}/$($this.Version)/python-$($this.Version)${architecture}${extension}"
return $uri
}
[string] Download() {
<#
.SYNOPSIS
Download Python installation executable into artifact location.
#>
$sourceUri = $this.GetSourceUri()
Write-Host "Sources URI: $sourceUri"
$sourcesLocation = Download-File -Uri $sourceUri -OutputFolder $this.WorkFolderLocation
Write-Debug "Done; Sources location: $sourcesLocation"
return $sourcesLocation
}
[void] CreateInstallationScript() {
<#
.SYNOPSIS
Create Python artifact installation script based on specified template.
#>
$sourceUri = $this.GetSourceUri()
$pythonExecName = [IO.path]::GetFileName($sourceUri.AbsoluteUri)
$installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath $this.InstallationTemplateName
$installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
$installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName -ItemType File
$variablesToReplace = @{
"{{__ARCHITECTURE__}}" = $this.Architecture;
"{{__VERSION__}}" = $this.Version;
"{{__PYTHON_EXEC_NAME__}}" = $pythonExecName
}
$variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
$installationTemplateContent | Out-File -FilePath $installationScriptLocation
Write-Debug "Done; Installation script location: $installationScriptLocation)"
}
[void] ArchiveArtifact() {
$OutputPath = Join-Path $this.ArtifactFolderLocation $this.OutputArtifactName
Create-SevenZipArchive -SourceFolder $this.WorkFolderLocation -ArchivePath $OutputPath
}
[void] Build() {
<#
.SYNOPSIS
Generates Python artifact from downloaded Python installation executable.
#>
Write-Host "Download Python $($this.Version) [$($this.Architecture)] executable..."
$this.Download()
Write-Host "Create installation script..."
$this.CreateInstallationScript()
Write-Host "Archive artifact"
$this.ArchiveArtifact()
}
}