Merge pull request #81 from actions/update-versions-manifest-file

[versions-manifest] Update for release from 02/04/2021

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @actions/virtual-environments-owners

.github/workflows/create-pr.yml

@@ -0,0 +1,33 @@
+name: Create Pull Request
+on:
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  create_pr:
+    name: Create Pull Request
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+
+    - name: Create versions-manifest.json
+      run: |
+        ./helpers/packages-generation/manifest-generator.ps1 -RepositoryFullName "$env:GITHUB_REPOSITORY" `
+                                                             -GitHubAccessToken "${{secrets.GITHUB_TOKEN}}" `
+                                                             -OutputFile "./versions-manifest.json" `
+                                                             -ConfigurationFile "./config/python-manifest-config.json"
+    - name: Create GitHub PR
+      run: |
+        $formattedDate = Get-Date -Format "MM/dd/yyyy"
+        ./helpers/github/create-pull-request.ps1 `
+                            -RepositoryFullName "$env:GITHUB_REPOSITORY" `
+                            -AccessToken "${{secrets.GITHUB_TOKEN}}" `
+                            -BranchName "update-versions-manifest-file" `
+                            -CommitMessage "Update versions-manifest" `
+                            -PullRequestTitle "[versions-manifest] Update for release from ${formattedDate}" `
+                            -PullRequestBody "Update versions-manifest.json for release from ${formattedDate}"

.github/workflows/create-release.yml

@@ -0,0 +1,18 @@
+name: Create release
+on:
+  repository_dispatch:
+    types: [create-release]
+
+jobs:
+  create_release:
+    name: Create release ${{ github.event.client_payload.ToolVersion }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Create release for Python ${{ github.event.client_payload.ToolVersion }}
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ github.event.client_payload.TagName }}
+        release_name: ${{ github.event.client_payload.ToolVersion }}
+        body: ${{ github.event.client_payload.ReleaseBody }}

.github/workflows/manifest-config-validation.yml

@@ -0,0 +1,32 @@
+name: Manifest config tests
+on: 
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - 'versions-manifest.json'
+
+jobs:
+  RunTests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: true
+
+    - name: Install Pester
+      shell: pwsh
+      run: |
+        Install-Module Pester -Force -Scope CurrentUser
+
+    - name: Run tests
+      shell: pwsh  
+      run: |
+        Import-Module Pester
+        Invoke-Pester -Configuration @{
+          Run = @{
+            Path = "tests/ManifestConfig.Tests.ps1"
+            Exit = $true
+          }
+        }

.github/workflows/releases-validation.yml

@@ -0,0 +1,35 @@
+name: Validate 'versions-manifest.json' file
+on: 
+  push:
+    branches:
+      - main
+    paths:
+      - 'versions-manifest.json'
+
+jobs:
+  setup-versions-from-manifest:
+    name: Setup ${{ matrix.python }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-16.04, ubuntu-18.04, ubuntu-20.04]
+        python: [3.5.4, 3.6.7, 3.7.5, 3.8.1]
+    steps:
+    - name: setup-python ${{ matrix.python }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Validate version
+      run: |
+        $pythonVersion = (python --version)
+        if ("Python ${{ matrix.python }}" -ne "$pythonVersion"){
+          Write-Host "The current version is $pythonVersion; expected version is ${{ matrix.python }}"
+          exit 1
+        }
+        $pythonVersion
+      shell: pwsh
+
+    - name: Run simple code
+      run: python -c 'import math; print(math.factorial(5))'

.github/workflows/test-python-version.yml

@@ -0,0 +1,34 @@
+name: Test Python version
+on: 
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Python version to build'
+        required: true
+      architecture:
+        description: 'The target architecture (x86, x64) of the Python'
+        required: false
+        default: 'x64'
+
+jobs:
+  test-python:
+    name: Test Python ${{ github.event.inputs.version }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-18.04, ubuntu-20.04] 
+    steps:
+    - name: Setup Python ${{ github.event.inputs.version }}
+      uses: actions/setup-python@main
+      with:
+        python-version: ${{ github.event.inputs.version }}
+        architecture: ${{ github.event.inputs.architecture }}
+
+    - name: Validate version
+      run: |
+        python --version
+      shell: pwsh
+
+    - name: Run simple code
+      run: python -c 'import math; print(math.factorial(5))'

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "helpers"]
 	path = helpers
 	url = https://github.com/actions/versions-package-tools
-	branch = master
+	branch = main

README.md

@@ -10,7 +10,7 @@ Some versions are pre-installed on [virtual-environments](https://github.com/act
 More versions will (soon!) be available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
 
 ## Adding new versions
-We are trying to build and release new versions of Python as soon as they are released. Please open an issue if any versions are missing.
+We are trying to build and release new versions of Python as soon as they are released. Please open an issue in [actions/virtual-environments](https://github.com/actions/virtual-environments) if any versions are missing.
 
 ## Contribution
 Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure

azure-pipelines/build-python-packages.yml

@@ -116,3 +116,17 @@ stages:
     Architecture: x86
   jobs:
   - template: /azure-pipelines/templates/test-job.yml
+
+- stage: Publish_Release
+  dependsOn: [Test_Python_MacOS, Test_Python_Ubuntu_1604, Test_Python_Ubuntu_1804, Test_Python_Ubuntu_2004, Test_Python_x64_Windows, Test_Python_x86_Windows]
+  jobs:
+  - deployment: Publish_Release
+    pool:
+      name: Azure Pipelines
+      vmImage: ubuntu-18.04
+    environment: 'Get Available Tools Versions - Publishing Approval'
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+          - template: /azure-pipelines/templates/publish-release-steps.yml

azure-pipelines/run-ci-builds.yml

@@ -3,7 +3,7 @@ pr:
   autoCancel: true
   branches:
     include:
-    - master
+    - main
   paths:
     exclude:
     - versions-manifest.json
@@ -29,6 +29,10 @@ jobs:
           -AzureDevOpsProjectName $(System.TeamProject) `
           -AzureDevOpsAccessToken $(System.AccessToken) `
           -SourceBranch $(Build.SourceBranch) `
+          -DefinitionId $(DEFINITION_ID) `
           -SourceVersion $(Build.SourceVersion) `
+          -ManifestLink $(MANIFEST_LINK) `
+          -WaitForBuilds $(WAIT_FOR_BUILDS) `
           -ToolVersions "$(PYTHON_VERSIONS)" `
-          -DefinitionId $(DEFINITION_ID) 
+          -RetryIntervalSec $(RETRY_INTERVAL_SEC) `
+          -RetryCount $(RETRY_COUNT)

azure-pipelines/templates/build-job.yml

@@ -7,13 +7,14 @@ jobs:
   steps:
   - checkout: self
     submodules: true
-    
+
   - task: PowerShell@2
     displayName: 'Build Python $(VERSION)'
     inputs:
       targetType: filePath
       filePath: './builders/build-python.ps1'
       arguments: '-Version $(VERSION) -Platform $(Platform) -Architecture $(Architecture)'
+      pwsh: true
 
   - task: PublishPipelineArtifact@1
     displayName: 'Publish Artifact: Python $(VERSION)'

azure-pipelines/templates/publish-release-steps.yml

@@ -0,0 +1,47 @@
+steps:
+- download: none
+
+- checkout: self
+  submodules: true
+
+- task: DownloadPipelineArtifact@2
+  inputs:
+    source: 'current'
+    path: $(Build.BinariesDirectory)
+
+- task: PowerShell@2
+  displayName: 'Create release Python $(VERSION)'
+  inputs:
+    TargetType: inline
+    script: |
+      $tagName = "$(VERSION)-$(Build.BuildId)"
+      $releaseBody = "Python $(VERSION)"
+      ./helpers/github/create-release.ps1 -RepositoryFullName "$(Build.Repository.Name)" `
+                                          -AccessToken "$(GITHUB_TOKEN)" `
+                                          -ToolVersion "$(VERSION)" `
+                                          -TagName "$tagName" `
+                                          -ReleaseBody "$releaseBody" `
+                                          -EventType "$(EVENT_TYPE)"
+- task: GitHubRelease@1
+  displayName: 'Upload release assets'
+  inputs:
+    gitHubConnection: 'Github Connection'
+    action: edit
+    tag: '$(VERSION)-$(Build.BuildId)'
+    title: '$(VERSION)'
+    releaseNotesSource: inline
+    releaseNotesInline: '$(RELEASE_NOTES_CONTENT)'
+    assets: '$(Build.BinariesDirectory)/*/*'
+    assetUploadMode: replace
+    addChangeLog: false
+
+- task: PowerShell@2
+  displayName: 'Trigger "Create Pull Request" workflow'
+  inputs:
+    TargetType: inline
+    script: |
+      Import-Module (Join-Path (Get-Location).Path "github-api.psm1")
+      $gitHubApi = Get-GitHubApi -RepositoryFullName "$(Build.Repository.Name)" -AccessToken "$(GITHUB_TOKEN)"
+      $gitHubApi.CreateWorkflowDispatch("$(WORKFLOW_FILE_NAME)", "$(WORKFLOW_DISPATCH_REF)", "$(INPUTS)")
+      Write-Host "Please find created Pull request here: $(Build.Repository.Uri)/pulls"
+    workingDirectory: '$(Build.SourcesDirectory)/helpers/github'

azure-pipelines/templates/test-job.yml

@@ -3,6 +3,8 @@ jobs:
   pool: 
     name: Azure Pipelines
     vmImage: $(VmImage)
+  variables:
+    TestRunTitle: 'python-$(Platform)-$(Architecture)'
   steps:
   - checkout: self
     submodules: true
@@ -25,6 +27,7 @@ jobs:
       archiveFilePatterns: '$(Build.BinariesDirectory)/python-$(VERSION)-$(Platform)-$(Architecture).*'
       destinationFolder: $(Build.BinariesDirectory)
       cleanDestinationFolder: false
+      overwriteExistingFiles: true
 
   - task: PowerShell@2
     displayName: 'Apply build artifact to the local machines'
@@ -74,6 +77,7 @@ jobs:
           }
         }
         Invoke-Pester -Script $pesterParams -OutputFile "test_results.xml" -OutputFormat NUnitXml
+      pwsh: true
       workingDirectory: '$(Build.SourcesDirectory)/tests'
 
   - task: PublishTestResults@2

builders/build-python.ps1

@@ -1,6 +1,6 @@
-using module "./builders/win-python-builder.psm1"
+using module "./win-python-builder.psm1"
-using module "./builders/ubuntu-python-builder.psm1"
+using module "./ubuntu-python-builder.psm1"
-using module "./builders/macos-python-builder.psm1"
+using module "./macos-python-builder.psm1"
 
 <#
 .SYNOPSIS
@@ -21,7 +21,7 @@ Required parameter. The platform for which Python will be built.
 #>
 
 param(
-    [Parameter (Mandatory=$true)][Version] $Version,
+    [Parameter (Mandatory=$true)][semver] $Version,
     [Parameter (Mandatory=$true)][string] $Platform,
     [string] $Architecture = "x64"
 )
@@ -29,6 +29,7 @@ param(
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "common-helpers.psm1") -DisableNameChecking
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "nix-helpers.psm1") -DisableNameChecking
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "win-helpers.psm1") -DisableNameChecking
+Import-Module (Join-Path $PSScriptRoot "python-version.psm1") -DisableNameChecking
 
 function Get-PythonBuilder {
     <#
@@ -49,13 +50,12 @@ function Get-PythonBuilder {
 
     #>
 
-    param (
+    param(
-        [version] $Version,
+        [semver] $Version,
         [string] $Architecture,
         [string] $Platform
     )
 
-    $Platform = $Platform.ToLower()  
     if ($Platform -match 'win32') {
         $builder = [WinPythonBuilder]::New($Version, $Architecture, $Platform)
     } elseif ($Platform -match 'linux') {
@@ -71,5 +71,5 @@ function Get-PythonBuilder {
 }
 
 ### Create Python builder instance, and build artifact
-$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture  -Platform $Platform 
+$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture -Platform $Platform 
 $Builder.Build()

builders/macos-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/nix-python-builder.psm1"
+using module "./nix-python-builder.psm1"
 
 class macOSPythonBuilder : NixPythonBuilder {
     <#
@@ -17,7 +17,7 @@ class macOSPythonBuilder : NixPythonBuilder {
     #>
 
     macOSPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) { }
@@ -29,17 +29,29 @@ class macOSPythonBuilder : NixPythonBuilder {
         #>
 
         $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation()
-        $configureString = "./configure --prefix=$pythonBinariesLocation --enable-optimizations --enable-shared --with-lto"
+        $configureString = "./configure"
+        $configureString += " --prefix=$pythonBinariesLocation"
+        $configureString += " --enable-optimizations"
+        $configureString += " --enable-shared"
+        $configureString += " --with-lto"
 
         ### OS X 10.11, Apple no longer provides header files for the deprecated system version of OpenSSL.
         ### Solution is to install these libraries from a third-party package manager,
         ### and then add the appropriate paths for the header and library files to configure command.
         ### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies)
         if ($this.Version -lt "3.7.0") {
-            $env:LDFLAGS="-L$(brew --prefix openssl)/lib"
+            $env:LDFLAGS = "-L/usr/local/opt/openssl@1.1/lib"
-            $env:CFLAGS="-I$(brew --prefix openssl)/include"
+            $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include"
         } else {
-            $configureString += " --with-openssl=/usr/local/opt/openssl"
+            $configureString += " --with-openssl=/usr/local/opt/openssl@1.1"
+        }
+
+        ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
+        ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
+        if ($this.Version -ge "3.2.0") {
+            $configureString += " --enable-loadable-sqlite-extensions"
+            $env:LDFLAGS += " -L$(brew --prefix sqlite3)/lib"
+            $env:CFLAGS += " -I$(brew --prefix sqlite3)/include"
         }
 
         Execute-Command -Command $configureString

builders/nix-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/python-builder.psm1"
+using module "./python-builder.psm1"
 
 class NixPythonBuilder : PythonBuilder {
     <#
@@ -33,13 +33,12 @@ class NixPythonBuilder : PythonBuilder {
     [string] $OutputArtifactName
 
     NixPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) {
         $this.InstallationTemplateName = "nix-setup-template.sh"	
         $this.InstallationScriptName = "setup.sh"
-
         $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.tar.gz"
     }
 
@@ -50,8 +49,10 @@ class NixPythonBuilder : PythonBuilder {
         #>
 
         $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $nativeVersion = Convert-Version -version $this.Version
 
-        return "${base}/$($this.Version)/Python-$($this.Version).tgz"
+        return "${base}/${versionName}/Python-${nativeVersion}.tgz"
     }
 
     [string] GetPythonBinary() {
@@ -95,9 +96,7 @@ class NixPythonBuilder : PythonBuilder {
         $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
 
         $variablesToReplace = @{
-            "{{__VERSION_MAJOR__}}" = $this.Version.Major;
+            "{{__VERSION_FULL__}}" = $this.Version;
-            "{{__VERSION_MINOR__}}" = $this.Version.Minor;
-            "{{__VERSION_BUILD__}}" = $this.Version.Build;
         }
         $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
 

builders/python-builder.psm1

@@ -29,7 +29,7 @@ class PythonBuilder {
 
     #>
 
-    [version] $Version
+    [semver] $Version
     [string] $Architecture
     [string] $Platform
     [string] $HostedToolcacheLocation
@@ -38,17 +38,17 @@ class PythonBuilder {
     [string] $ArtifactFolderLocation
     [string] $InstallationTemplatesLocation
 
-    PythonBuilder ([version] $version, [string] $architecture, [string] $platform) {
+    PythonBuilder ([semver] $version, [string] $architecture, [string] $platform) {
-        $this.Version = $version
+        $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers"
-        $this.Architecture = $architecture
-        $this.Platform = $platform
 
         $this.HostedToolcacheLocation = $env:AGENT_TOOLSDIRECTORY
         $this.TempFolderLocation = $env:BUILD_SOURCESDIRECTORY
         $this.WorkFolderLocation = $env:BUILD_BINARIESDIRECTORY
         $this.ArtifactFolderLocation = $env:BUILD_STAGINGDIRECTORY
 
-        $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers"
+        $this.Version = $version
+        $this.Architecture = $architecture
+        $this.Platform = $platform
     }
 
     [uri] GetBaseUri() {
@@ -79,11 +79,21 @@ class PythonBuilder {
         return "$pythonToolcacheLocation/$($this.Version)/$($this.Architecture)"
     }
 
+    [string] GetBaseVersion() {
+        <#
+        .SYNOPSIS
+        Return Major.Minor.Patch version string.
+        #>
+
+        return "$($this.Version.Major).$($this.Version.Minor).$($this.Version.Patch)"
+    }
+
     [void] PreparePythonToolcacheLocation() {
         <#
         .SYNOPSIS
         Prepare system hostedtoolcache folder for new Python version. 
         #>
+        
         $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation()
 
         if (Test-Path $pythonBinariesLocation) {

builders/python-version.psm1

@@ -0,0 +1,45 @@
+function Convert-Label() {
+    <#
+    .SYNOPSIS
+    Convert generic semver label to native Python label.
+    #>
+
+    param(
+        [Parameter(Mandatory)]
+        [string] $label
+    )
+
+    switch ($label) {
+        "alpha" { return "a" }
+        "beta" { return "b" }
+        "rc" { return "rc" }
+        default { throw "Invalid version label '$label'" }
+    }
+}
+
+function Convert-Version {
+    <#
+    .SYNOPSIS
+    Convert generic semver version to native Python version.
+    #>
+
+    param(
+        [Parameter(Mandatory)]
+        [semver] $version,
+        [char] $delimiter = "."
+    )
+
+    $nativeVersion = "{0}.{1}.{2}" -f $version.Major, $version.Minor, $version.Patch
+
+    if ($version.PreReleaseLabel)
+    {
+        $preReleaseLabel = $version.PreReleaseLabel.Split($delimiter)
+        
+        $preReleaseLabelName = Convert-Label -Label $preReleaseLabel[0]
+        $preReleaseLabelVersion = $preReleaseLabel[1]
+
+        $nativeVersion += "${preReleaseLabelName}${preReleaseLabelVersion}"
+    }
+
+    return $nativeVersion
+}

builders/ubuntu-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/nix-python-builder.psm1"
+using module "./nix-python-builder.psm1"
 
 class UbuntuPythonBuilder : NixPythonBuilder {
     <#
@@ -17,7 +17,7 @@ class UbuntuPythonBuilder : NixPythonBuilder {
     #>
 
     UbuntuPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) { }
@@ -32,13 +32,22 @@ class UbuntuPythonBuilder : NixPythonBuilder {
 
         ### To build Python with SO we must pass full path to lib folder to the linker
         $env:LDFLAGS="-Wl,--rpath=${pythonBinariesLocation}/lib"
-        $configureString = "./configure --prefix=$pythonBinariesLocation --enable-shared --enable-optimizations"
+        $configureString = "./configure"
+        $configureString += " --prefix=$pythonBinariesLocation"
+        $configureString += " --enable-shared"
+        $configureString += " --enable-optimizations"
 
+        ### Compile with ucs4 for Python 2.x. On 3.x, ucs4 is enabled by default
         if ($this.Version -lt "3.0.0") {
-            ### Compile with ucs4 for Python 2.x. On 3.x, ucs4 is enabled by default
             $configureString += " --enable-unicode=ucs4"
         }
 
+        ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
+        ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
+        if ($this.Version -ge "3.2.0") {
+            $configureString += " --enable-loadable-sqlite-extensions"
+        }
+
         Execute-Command -Command $configureString
     }
 
@@ -59,6 +68,7 @@ class UbuntuPythonBuilder : NixPythonBuilder {
         } else {
             $tkinterInstallString = "sudo apt install -y python-tk tk-dev"
         }
+
         Execute-Command -Command $tkinterInstallString
 
         ### Install dependent packages

builders/win-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/python-builder.psm1"
+using module "./python-builder.psm1"
 
 class WinPythonBuilder : PythonBuilder {
     <#
@@ -27,7 +27,7 @@ class WinPythonBuilder : PythonBuilder {
     [string] $OutputArtifactName
 
     WinPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) {
@@ -72,10 +72,12 @@ class WinPythonBuilder : PythonBuilder {
         #>
 
         $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $nativeVersion = Convert-Version -version $this.Version
         $architecture = $this.GetArchitectureExtension()
         $extension = $this.GetPythonExtension()
 
-        $uri = "${base}/$($this.Version)/python-$($this.Version)${architecture}${extension}"
+        $uri = "${base}/${versionName}/python-${nativeVersion}${architecture}${extension}"
 
         return $uri
     }

config/python-manifest-config.json

@@ -1,8 +1,8 @@
 {
-    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+)-([\\w\\.]+)?-?(x\\d+)",
+    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?(x\\d+)",
     "groups": {
-        "arch": 3,
+        "arch": 4,
-        "platform": 1,
+        "platform": 2,
-        "platform_version": 2
+        "platform_version": 3
     }
 }

installers/nix-setup-template.sh

@@ -1,13 +1,12 @@
 set -e
 
-MAJOR_VERSION="{{__VERSION_MAJOR__}}"
+PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
-MINOR_VERSION="{{__VERSION_MINOR__}}"
+MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
-BUILD_VERSION="{{__VERSION_BUILD__}}"
+MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
 
 PYTHON_MAJOR=python$MAJOR_VERSION
 PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION
 PYTHON_MAJORMINOR=python$MAJOR_VERSION$MINOR_VERSION
-PYTHON_FULL_VERSION=$MAJOR_VERSION.$MINOR_VERSION.$BUILD_VERSION
 
 if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then
     # No AGENT_TOOLSDIRECTORY on GitHub images
@@ -51,7 +50,7 @@ chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR pyth
 
 echo "Upgrading PIP..."
 ./python -m ensurepip
-./python -m pip install --ignore-installed pip
+./python -m pip install --ignore-installed pip --no-warn-script-location
 
 echo "Create complete file"
 touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete

installers/win-setup-template.ps1

@@ -1,10 +1,9 @@
 [String] $Architecture = "{{__ARCHITECTURE__}}"
-[Version] $Version = "{{__VERSION__}}"
+[String] $Version = "{{__VERSION__}}"
 [String] $PythonExecName = "{{__PYTHON_EXEC_NAME__}}"
 
 function Get-RegistryVersionFilter {
-    param
+    param(
-    (
         [Parameter(Mandatory)][String] $Architecture,
         [Parameter(Mandatory)][Int32] $MajorVersion,
         [Parameter(Mandatory)][Int32] $MinorVersion
@@ -12,20 +11,15 @@ function Get-RegistryVersionFilter {
 
     $archFilter = if ($Architecture -eq 'x86') { "32-bit" } else { "64-bit" }
     ### Python 2.7 x86 have no architecture postfix
-    if (($Architecture -eq "x86") -and ($MajorVersion -eq 2))
+    if (($Architecture -eq "x86") -and ($MajorVersion -eq 2)) {
-    {
         "Python $MajorVersion.$MinorVersion.\d+$"
-    }
+    } else {
-    else
-    {
         "Python $MajorVersion.$MinorVersion.*($archFilter)"
     }
 }
 
-function Remove-RegistryEntries
+function Remove-RegistryEntries {
-{
+    param(
-    param
-    (
         [Parameter(Mandatory)][String] $Architecture,
         [Parameter(Mandatory)][Int32] $MajorVersion,
         [Parameter(Mandatory)][Int32] $MinorVersion
@@ -35,10 +29,8 @@ function Remove-RegistryEntries
 
     $regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products"
     $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName
-    foreach ($key in $regKeys)
+    foreach ($key in $regKeys) {
-    {
+        if ($key.getValue("DisplayName") -match $versionFilter) {
-        if ($key.getValue("DisplayName") -match $versionFilter)
-        {
             Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose
         }
     }
@@ -63,40 +55,34 @@ function Remove-RegistryEntries
 }
 
 function Get-ExecParams {
-    param
+    param(
-    (
         [Parameter(Mandatory)][Boolean] $IsMSI,
         [Parameter(Mandatory)][String] $PythonArchPath
     )
 
-    if ($IsMSI) 
+    if ($IsMSI) {
-    {
         "TARGETDIR=$PythonArchPath ALLUSERS=1"
-    } 
+    } else {
-    else 
-    {
         "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1"
     }
 }
 
 $ToolcacheRoot = $env:AGENT_TOOLSDIRECTORY
-if ([string]::IsNullOrEmpty($ToolcacheRoot))
+if ([string]::IsNullOrEmpty($ToolcacheRoot)) {
-{
     # GitHub images don't have `AGENT_TOOLSDIRECTORY` variable
     $ToolcacheRoot = $env:RUNNER_TOOL_CACHE
 }
 $PythonToolcachePath = Join-Path -Path $ToolcacheRoot -ChildPath "Python"
-$PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version.ToString()
+$PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version
 $PythonArchPath = Join-Path -Path $PythonVersionPath -ChildPath $Architecture
 
 $IsMSI = $PythonExecName -match "msi"
 
-$MajorVersion = $Version.Major
+$MajorVersion = $Version.Split('.')[0]
-$MinorVersion = $Version.Minor
+$MinorVersion = $Version.Split('.')[1]
 
 Write-Host "Check if Python hostedtoolcache folder exist..."
-if (-Not (Test-Path $PythonToolcachePath)) 
+if (-Not (Test-Path $PythonToolcachePath)) {
-{
     Write-Host "Create Python toolcache folder"
     New-Item -ItemType Directory -Path $PythonToolcachePath | Out-Null
 }
@@ -104,22 +90,17 @@ if (-Not (Test-Path $PythonToolcachePath))
 Write-Host "Check if current Python version is installed..."
 $InstalledVersions = Get-Item "$PythonToolcachePath\$MajorVersion.$MinorVersion.*\$Architecture"
 
-if ($null -ne $InstalledVersions)
+if ($null -ne $InstalledVersions) {
-{
     Write-Host "Python$MajorVersion.$MinorVersion ($Architecture) was found in $PythonToolcachePath..."
 
-    foreach ($InstalledVersion in $InstalledVersions)
+    foreach ($InstalledVersion in $InstalledVersions) {
-    {
+        if (Test-Path -Path $InstalledVersion) {
-        if (Test-Path -Path $InstalledVersion)
-        {
             Write-Host "Deleting $InstalledVersion..."
             Remove-Item -Path $InstalledVersion -Recurse -Force
             Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose
         }
     }
-} 
+} else {
-else 
-{
     Write-Host "No Python$MajorVersion.$MinorVersion.* found"
 }
 
@@ -136,14 +117,18 @@ Write-Host "Install Python $Version in $PythonToolcachePath..."
 $ExecParams = Get-ExecParams -IsMSI $IsMSI -PythonArchPath $PythonArchPath
 
 cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet"
-if ($LASTEXITCODE -ne 0)
+if ($LASTEXITCODE -ne 0) {
-{
     Throw "Error happened during Python installation"
 }
 
+Write-Host "Create `python3` symlink"
+if ($MajorVersion -ne "2") {
+    New-Item -Path "$PythonArchPath\python3.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python.exe"
+}
+
 Write-Host "Install and upgrade Pip"
 $PythonExePath = Join-Path -Path $PythonArchPath -ChildPath "python.exe"
-cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip"
+cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip --no-warn-script-location"
 
 Write-Host "Create complete file"
 New-Item -ItemType File -Path $PythonVersionPath -Name "$Architecture.complete" | Out-Null

tests/ManifestConfig.Tests.ps1

@@ -0,0 +1,44 @@
+Import-Module (Join-Path $PSScriptRoot "../helpers/packages-generation/manifest-utils.psm1")
+
+$ConfigurationFile = Join-Path $PSScriptRoot "../config/python-manifest-config.json"
+$Configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
+
+$stableTestCases = @(
+    @{ ReleaseName = "python-3.8.3-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-linux-16.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "16.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
+) | ForEach-Object { $_.Configuration = $Configuration; $_ }
+
+$unstableTestCases = @(
+    @{ ReleaseName = "python-3.9.0-alpha.2-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-beta.1-linux-16.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "16.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-rc.4-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-beta.2-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-beta.2-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-beta.2-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
+) | ForEach-Object { $_.Configuration = $Configuration; $_ }
+
+Describe "Python manifest config" {
+    Context "Stable versions" {
+        It "<ReleaseName>" -TestCases $stableTestCases {
+            $Release = @{ name = $ReleaseName }
+            $asset = New-AssetItem -ReleaseAsset $Release -Configuration $Configuration
+            $asset.platform | Should -Be $ExpectedResult.platform
+            $asset.platform_version | Should -Be $ExpectedResult.platform_version
+            $asset.arch | Should -Be $ExpectedResult.arch
+        }
+    }
+
+    Context "Prerelease versions" {
+        It "<ReleaseName>" -TestCases $unstableTestCases {
+            $Release = @{ name = $ReleaseName }
+            $asset = New-AssetItem -ReleaseAsset $Release -Configuration $Configuration
+            $asset.platform | Should -Be $ExpectedResult.platform
+            $asset.platform_version | Should -Be $ExpectedResult.platform_version
+            $asset.arch | Should -Be $ExpectedResult.arch
+        }
+    }
+}

tests/python-tests.ps1

@@ -1,12 +1,13 @@
 param (
-    [Version] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    [semver] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
     $Version,
-    [String] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
     $Platform
 )
 
 Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
 Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1")
+Import-Module (Join-Path $PSScriptRoot "../builders/python-version.psm1")
 
 function Analyze-MissingModules([string] $buildOutputLocation) {
     $searchStringStart = "Failed to build these modules:"
@@ -41,6 +42,17 @@ Describe "Tests" {
         "python ./sources/simple-test.py" | Should -ReturnZeroExitCode
     }
 
+    if ($Version -ge "3.2.0") {
+        It "Check if sqlite3 module is installed" {
+            "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
+        }
+    }
+
+    It "Run pip" {
+        "pip install requests" | Should -ReturnZeroExitCode
+        "pip uninstall requests -y" | Should -ReturnZeroExitCode
+    }
+
     if (IsNixPlatform $Platform) {
 
         It "Check for failed modules in build_output" {
@@ -53,7 +65,8 @@ Describe "Tests" {
         }
 
         It "Check if python configuration is correct" {
-            "python ./sources/python-config-test.py" | Should -ReturnZeroExitCode
+            $nativeVersion = Convert-Version -version $Version
+            "python ./sources/python-config-test.py $Version $nativeVersion" | Should -ReturnZeroExitCode
         }
 
         It "Check if shared libraries are linked correctly" {
@@ -62,7 +75,7 @@ Describe "Tests" {
     }
 
     # Pyinstaller 3.5 does not support Python 3.8.0. Check issue https://github.com/pyinstaller/pyinstaller/issues/4311
-    if ($Version -lt "3.8.0") {
+    if ($Version -lt "3.8.0" -and $Version.Major -ne "2") {
         It "Validate Pyinstaller" {
             "pip install pyinstaller" | Should -ReturnZeroExitCode
             "pyinstaller --onefile ./sources/simple-test.py" | Should -ReturnZeroExitCode

tests/sources/python-config-test.py

@@ -1,4 +1,5 @@
 import distutils.sysconfig
+from distutils.version import LooseVersion
 import sysconfig
 import sys
 import platform
@@ -6,7 +7,8 @@ import os
 
 # Define variables
 os_type = platform.system()
-version = sys.version.split(" ")[0]
+version = sys.argv[1]
+nativeVersion = sys.argv[2]
 
 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
@@ -41,7 +43,7 @@ else:
 ### Validate macOS
 if os_type == 'Darwin':
     ### Validate openssl links
-    if version < "3.7.0":
+    if LooseVersion(nativeVersion) < LooseVersion("3.7.0"):
         expected_ldflags = '-L/usr/local/opt/openssl@1.1/lib'
         ldflags = sysconfig.get_config_var('LDFLAGS')
 
@@ -49,8 +51,8 @@ if os_type == 'Darwin':
             print('Invalid ldflags: %s; Expected: %s' % (ldflags, expected_ldflags))
             exit(1)
     else:
-        expected_openssl_includes = '-I/usr/local/opt/openssl/include'
+        expected_openssl_includes = '-I/usr/local/opt/openssl@1.1/include'
-        expected_openssl_ldflags ='-L/usr/local/opt/openssl/lib'
+        expected_openssl_ldflags ='-L/usr/local/opt/openssl@1.1/lib'
         
         openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES')
         openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS')

tests/sources/python-modules.py

@@ -139,6 +139,7 @@ standard_library = [
     'sre_constants',
     'sre_parse',
     'ssl',
+    '_ssl',
     'stat',
     'string',
     'stringprep',
@@ -251,6 +252,15 @@ if sys.version_info >= (3, 7):
 if sys.version_info > (3, 7):
     standard_library.remove('macpath')
 
+# 'dummy_threading' module has been removed from Python 3.9
+if sys.version_info > (3, 8):
+    standard_library.remove('dummy_threading')
+
+# 'symbol' and 'formatter' modules have been removed from Python 3.10
+if sys.version_info >= (3, 10):
+    standard_library.remove('symbol')
+    standard_library.remove('formatter')
+
 # Remove tkinter and Easter eggs
 excluded_modules = [
     'antigravity',
@@ -261,7 +271,6 @@ excluded_modules = [
 def check_missing_modules(expected_modules):
     missing = []
     for module in expected_modules:
-        print('Try to import module ', module)
         try:
             importlib.import_module(module)
         except:

tests/sources/python-sqlite3.py

@@ -0,0 +1,19 @@
+import sqlite3
+from sqlite3 import Error
+
+def create_connection(db_file):
+    """ create a database connection to a SQLite database """
+    conn = None
+    try:
+        print('Sqlite3 version: ', sqlite3.version)
+        conn = sqlite3.connect(db_file)
+        conn.enable_load_extension(True)
+    except Error as e:
+        print(e)
+        exit(1)
+    finally:
+        if conn:
+            conn.close()
+
+if __name__ == '__main__':
+    create_connection(r"pythonsqlite.db")