Update versions-manifest (#261)

Co-authored-by: Service account <no-reply@microsoft.com>

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @actions/setup-actions-team

.github/workflows/build-python-packages.yml

@@ -0,0 +1,233 @@
+name: Build Python package
+run-name: Generate Python ${{ inputs.VERSION || '3.11.0' }}
+on:
+  workflow_dispatch:
+    inputs:
+      VERSION:
+        description: 'Python version to build and upload'
+        default: '3.11.0'
+        required: true
+      PUBLISH_RELEASES:
+        description: 'Whether to publish releases'
+        required: true
+        type: boolean
+        default: false
+      PLATFORMS:
+        description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
+        required: true
+        default: 'ubuntu-20.04,ubuntu-22.04,macos-11_x64,macos-11_arm64,windows-2019_x64,windows-2019_x86'
+  pull_request:
+    paths-ignore:
+    - 'versions-manifest.json'
+    - 'LICENSE'
+    - '**.md'
+    branches:
+    - 'main'
+
+env:
+  VERSION: ${{ inputs.VERSION || '3.11.0' }}
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  generate_matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generate-matrix.outputs.matrix }}
+    steps:
+      - name: Generate execution matrix
+        id: generate-matrix
+        run: |
+          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,macos-11,macos-11_arm64,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
+          $matrix = @()
+
+          foreach ($configuration in $configurations) {
+            $parts = $configuration.Split("_")
+            $os = $parts[0]
+            $arch = if ($parts[1]) {$parts[1]} else {"x64"}
+            switch -wildcard ($os) {
+              "*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
+              "*macos*" { $platform = 'darwin' }
+              "*windows*" { $platform = 'win32' }
+            }
+            $matrix += @{
+              'platform' = $platform
+              'os' = $os
+              'arch' = $arch
+            }
+          }
+          echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT
+
+  build_python:
+    needs: generate_matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
+    runs-on: ${{ matrix.os }}
+    env:
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
+    steps:
+
+      - name: Check out repository code
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Build Python ${{ env.VERSION }}
+        run: |
+          ./builders/build-python.ps1 -Version $env:VERSION `
+                  -Platform ${{ matrix.platform }} -Architecture ${{ matrix.arch }}
+
+      - name: Publish artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ${{ runner.temp }}/artifact
+
+  test_python:
+    needs: [generate_matrix, build_python]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
+    runs-on: ${{ matrix.os }}
+    env:
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
+    steps:
+
+      - name: Check out repository code
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Fully cleanup the toolcache directory before testing
+        run: ./helpers/clean-toolcache.ps1 -ToolName "Python"
+
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ runner.temp }}
+
+      - name: Extract files
+        run: |
+          if ('${{ matrix.platform }}' -eq 'win32') {
+            $artifactName = "${{ env.ARTIFACT_NAME }}.zip"
+            7z.exe x "$artifactName" -y | Out-Null
+          } else {
+            $artifactName = "${{ env.ARTIFACT_NAME }}.tar.gz"
+            tar -xzf $artifactName
+          }
+        working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
+
+      - name: Apply build artifact to the local machine
+        run: |
+          if ('${{ matrix.platform }}' -eq 'win32') {
+            powershell ./setup.ps1
+            } else {
+              sh ./setup.sh
+              cp ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}/build_output.txt ${{ runner.temp }}
+            }
+        working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
+
+      - name: Setup Python ${{ env.VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.VERSION }}
+          architecture: ${{ matrix.arch }}
+
+      - name: Verbose sysconfig dump
+        if: runner.os == 'Linux' || runner.os == 'macOS'
+        run: python ./sources/python-config-output.py
+        working-directory: ${{ github.workspace }}/tests
+
+      - name: Verbose python binary links
+        if: runner.os == 'Linux' || runner.os == 'macOS'
+        run: |
+          $pythonLocation = which python
+          if ('${{ matrix.platform }}' -eq 'darwin') { otool -L $pythonLocation } else { ldd $pythonLocation }
+
+      - name: Run tests
+        run: |
+            Install-Module Pester -Force -Scope CurrentUser
+            Import-Module Pester
+            $pesterContainer = New-PesterContainer -Path './python-tests.ps1' -Data @{
+                Version="${{ env.VERSION }}";
+                Platform="${{ matrix.platform }}";
+                Architecture="${{ matrix.arch }}";
+            }
+            $Result = Invoke-Pester -Container $pesterContainer -PassThru
+            if ($Result.FailedCount -gt 0) {
+              $host.SetShouldExit($Result.FailedCount)
+              exit $Result.FailedCount
+            }
+        working-directory: ${{ github.workspace }}/tests
+
+  publish_release:
+      name: Publish release
+      if: github.event_name == 'workflow_dispatch' && inputs.PUBLISH_RELEASES
+      needs: test_python
+      runs-on: ubuntu-latest
+      steps:
+      - uses: actions/download-artifact@v3
+
+      - name: Publish Release ${{ env.VERSION }}
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.VERSION }}-${{ github.run_id }}
+          release_name: ${{ env.VERSION }}
+          body: |
+            Python ${{ env.VERSION }}
+
+      - name: Generate hash for packages
+        run: |
+          $childItems = Get-Childitem -Path '.'
+          $childItems | Foreach-Object {
+            $packageObj = Get-Childitem -Path $_.FullName | Select-Object -First 1
+            Write-Host "Package: $($packageObj.Name)"
+            $actualHash = (Get-FileHash -Path $packageObj.FullName -Algorithm sha256).Hash
+            $hashString = "$actualHash $($packageObj.Name)"
+            Write-Host "$hashString"
+            Add-Content -Path ./hashes.sha256 -Value "$hashString"
+          } 
+  
+      - name: Upload release assets
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            for (let artifactDir of fs.readdirSync('.')) {
+              let artifactName = fs.lstatSync(artifactDir).isDirectory() ? fs.readdirSync(`${artifactDir}`)[0] : artifactDir;
+  
+              console.log(`Upload ${artifactName} asset`);
+              github.rest.repos.uploadReleaseAsset({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                release_id: ${{ steps.create_release.outputs.id }},
+                name: artifactName,
+                data: fs.lstatSync(artifactDir).isDirectory() ? fs.readFileSync(`./${artifactDir}/${artifactName}`) : fs.readFileSync(`./${artifactName}`).toString()
+              });
+            }
+
+  trigger_pr:
+    name: Trigger "Create Pull Request" workflow
+    needs: publish_release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Trigger "Create Pull Request" workflow
+      uses: actions/github-script@v6
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          github.rest.actions.createWorkflowDispatch({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            workflow_id: 'create-pr.yml',
+            ref: 'main'
+          });
+

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,16 @@
+name: CodeQL analysis
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 3 * * 0'
+
+jobs:
+  call-codeQL-analysis:
+    name: CodeQL analysis 
+    uses: actions/reusable-workflows/.github/workflows/codeql-analysis.yml@main
+    with:
+     languages: "['python']"

.github/workflows/create-pr.yml

@@ -0,0 +1,10 @@
+name: Create Pull Request
+on:
+  workflow_dispatch:
+
+jobs:
+  create-pr:
+    uses: actions/versions-package-tools/.github/workflows/create-pr-to-update-manifest.yml@main
+    with:
+      tool-name: "python"
+    secrets: inherit

.github/workflows/create-release.yml

@@ -0,0 +1,18 @@
+name: Create release
+on:
+  repository_dispatch:
+    types: [create-release]
+
+jobs:
+  create_release:
+    name: Create release ${{ github.event.client_payload.ToolVersion }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Create release for Python ${{ github.event.client_payload.ToolVersion }}
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ github.event.client_payload.TagName }}
+        release_name: ${{ github.event.client_payload.ToolVersion }}
+        body: ${{ github.event.client_payload.ReleaseBody }}

.github/workflows/get-python-versions.yml

@@ -0,0 +1,13 @@
+name: Get Python versions
+on:
+  schedule:
+    - cron: '0 3,15 * * *'
+  workflow_dispatch:
+
+jobs:
+  get-new-python-versions:
+    uses: actions/versions-package-tools/.github/workflows/get-new-tool-versions.yml@main
+    with:
+      tool-name: "Python"
+      image-url: "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+    secrets: inherit

.github/workflows/manifest-config-validation.yml

@@ -0,0 +1,32 @@
+name: Manifest config tests
+on: 
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - 'versions-manifest.json'
+
+jobs:
+  RunTests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        submodules: true
+
+    - name: Install Pester
+      shell: pwsh
+      run: |
+        Install-Module Pester -Force -Scope CurrentUser
+
+    - name: Run tests
+      shell: pwsh
+      run: |
+        Import-Module Pester
+        Invoke-Pester -Configuration @{
+          Run = @{
+            Path = "tests/ManifestConfig.Tests.ps1"
+            Exit = $true
+          }
+        }

.github/workflows/python-versions-runner.yml

@@ -0,0 +1,31 @@
+name: Python versions runner
+on:
+  workflow_dispatch:
+    inputs:
+      versions:
+        description: 'Versions to build'
+        required: true
+        default: '","'
+      publish-releases:
+        description: 'Whether to publish releases'
+        required: true
+        default: 'false'
+
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  trigger_builds:
+    name: Trigger python build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      
+      - name: Trigger python workflow
+        run: |
+          $versions = ${{ github.event.inputs.versions }}
+          ./builders/python-versions-runner.ps1 -Versions $versions.Split(",") -PublishRelease ${{ github.event.inputs.publish-releases }}
+        env:
+          TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/releases-validation.yml

@@ -0,0 +1,36 @@
+name: Validate 'versions-manifest.json' file
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'versions-manifest.json'
+
+jobs:
+  setup-versions-from-manifest:
+    name: Setup ${{ matrix.python }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04]
+        python: [3.9.12, 3.10.8, 3.11.8]
+    steps:
+    - name: setup-python ${{ matrix.python }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Validate version
+      run: |
+        $pythonVersion = (python --version)
+        if ("Python ${{ matrix.python }}" -ne "$pythonVersion"){
+          Write-Host "The current version is $pythonVersion; expected version is ${{ matrix.python }}"
+          exit 1
+        }
+        $pythonVersion
+      shell: pwsh
+
+    - name: Run simple code
+      run: python -c 'import math; print(math.factorial(5))'

.github/workflows/test-python-version.yml

@@ -0,0 +1,34 @@
+name: Test Python version
+on: 
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Python version to build'
+        required: true
+      architecture:
+        description: 'The target architecture (x86, x64) of the Python'
+        required: false
+        default: 'x64'
+
+jobs:
+  test-python:
+    name: Test Python ${{ github.event.inputs.version }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04] 
+    steps:
+    - name: Setup Python ${{ github.event.inputs.version }}
+      uses: actions/setup-python@main
+      with:
+        python-version: ${{ github.event.inputs.version }}
+        architecture: ${{ github.event.inputs.architecture }}
+
+    - name: Validate version
+      run: |
+        python --version
+      shell: pwsh
+
+    - name: Run simple code
+      run: python -c 'import math; print(math.factorial(5))'

.github/workflows/validate-manifest.yml

@@ -0,0 +1,21 @@
+name: Validate manifest
+on:
+# The GITHUB_TOKEN secret is used to create a PR
+# The pull_request event will not be triggered by it
+# That's one of the reasons we need the schedule to validate the versions-manifest.json file
+  schedule:
+    - cron: '0 8,20 * * *'
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'versions-manifest.json'
+
+jobs:
+  manifest:
+    uses: actions/versions-package-tools/.github/workflows/validate-manifest.yml@main
+    with:
+      tool-name: "Python"
+      image-url: "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+    secrets: inherit

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "helpers"]
+	path = helpers
+	url = https://github.com/actions/versions-package-tools
+	branch = main

README.md

@@ -1,16 +1,27 @@
 # Python for Actions
-This repository contains the code and scripts that we use to build Python packages used in [virtual-environments](https://github.com/actions/virtual-environments) and accessible through the [setup-python](https://github.com/actions/setup-python) Action.  
-File [versions-manifest.json](./versions-manifest.json) contains the list of available and released versions.  
+This repository contains the code and scripts that we use to build Python packages used in [runner-images](https://github.com/actions/runner-images) and accessible through the [setup-python](https://github.com/actions/setup-python) Action.
+File [versions-manifest.json](./versions-manifest.json) contains the list of available and released versions.
 
-> Caution: this is prepared for and only permitted for use by actions `virtual-environments` and `setup-python` action.
+> Caution: this is prepared for and only permitted for use by actions `runner-images` and `setup-python` action.
 
-**Status**: Currently under development and in use for beta and preview actions.  This repo is undergoing rapid changes.
+Some versions are pre-installed on [runner-images](https://github.com/actions/runner-images) images.
+More versions are available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
 
-Some versions are pre-installed on [virtual-environments](https://github.com/actions/virtual-environments) images.  
-More versions will (soon!) be available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
+## Building installation packages
+**Ubuntu:** The official Python sources are downloaded from [python.org](https://www.python.org/ftp/python/), built using the `make` tool, and archived along with the installation script for further distribution and installation. We build each Python version against all [versions of Ubuntu](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) that are available at the time the Python version is released.
 
-## Adding new versions
-We are trying to build and release new versions of Python as soon as they are released. Please open an issue if any versions are missing.
+**macOS:** 
+- For **Python < 3.11**, the official Python sources are downloaded from [python.org](https://www.python.org/ftp/python/), built using the `make` tool, and archived along with the installation script for further distribution and installation.  For backward compatibility, we build each Python version against the oldest [version of macOS](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) that is available at the time the Python version is released.
+- For **Python >= 3.11**, the official macOS `universal2` Python binaries are simply downloaded from [python.org](https://www.python.org/ftp/python/) and archived along with the installation script for further distribution and installation.
+
+**Windows:** The official Python executables are simply downloaded from [python.org](https://www.python.org/ftp/python/) and archived along with the installation script for further distribution and installation.
+
+## Support policy
+We aim to make new versions of Python available as soon as they are released. Both stable and prerelease versions are considered for building and releasing. Please open an issue in [actions/setup-python](https://github.com/actions/setup-python) if a version you're looking for isn't available.
+
+When a new version of an operating system is released and made available for use with [GitHub hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources), we will provide the **latest existing patch versions of Python for all major versions that have not reached [end-of-life](https://devguide.python.org/versions/)**.  Any subsequent Python versions will be made available for the new OS as well.
+
+Beginning **approximately six months prior** to the removal of a Python version from the [versions-manifest.json](https://github.com/actions/python-versions/blob/main/versions-manifest.json) file, a pinned issue will be created in the [setup-python](https://github.com/actions/setup-python) repository. This pinned issue will provide important details about the upcoming end of support, including the specific date, as well as any other notes, relevant updates or alternatives. We encourage users to regularly check pinned issues for updates on tool versions they are using for maximum transparency, security, performance and overall compatibility with their projects.
 
 ## Contribution
 Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure

azure-pipelines/build-python-packages.yml

@@ -1,99 +0,0 @@
-name: $(date:yyyyMMdd)$(rev:.r)-Python-$(VERSION)
-trigger: none
-pr: none
-
-stages:
-- stage: Build_Python_MacOS
-  dependsOn: []
-  variables:
-    VmImage: 'macOS-10.14'
-    Platform: macos-1014
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_MacOS
-  condition: succeeded()
-  dependsOn: Build_Python_MacOS
-  variables:
-    VmImage: 'macOS-10.14'
-    Platform: macos-1014
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_Ubuntu_1604
-  dependsOn: []
-  variables:
-    VmImage: 'ubuntu-16.04'
-    Platform: ubuntu-1604
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_Ubuntu_1604
-  condition: succeeded()
-  dependsOn: Build_Python_Ubuntu_1604
-  variables:
-    VmImage: 'ubuntu-16.04'
-    Platform: ubuntu-1604
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_Ubuntu_1804
-  dependsOn: []
-  variables:
-    VmImage: 'ubuntu-18.04'
-    Platform: ubuntu-1804
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_Ubuntu_1804
-  condition: succeeded()
-  dependsOn: Build_Python_Ubuntu_1804
-  variables:
-    VmImage: 'ubuntu-18.04'
-    Platform: ubuntu-1804
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_X64_Windows
-  dependsOn: []
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: windows-2016
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_x64_Windows
-  condition: succeeded()
-  dependsOn: Build_Python_X64_Windows
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: windows-2016
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_x86_Windows
-  dependsOn: []
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: windows-2016
-    Architecture: x86
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_x86_Windows
-  condition: succeeded()
-  dependsOn: Build_Python_x86_Windows
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: windows-2016
-    Architecture: x86
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml

azure-pipelines/run-ci-builds.yml

@@ -1,33 +0,0 @@
-trigger: none
-pr:
-  autoCancel: true
-  branches:
-    include:
-    - master
-  paths:
-    exclude:
-    - versions-manifest.json
-
-jobs:
-- job: Run_Builds
-  pool:
-    name: Azure Pipelines
-    vmImage: 'ubuntu-latest'
-
-  steps:
-  - checkout: self
-    submodules: true
-
-  - task: PowerShell@2
-    displayName: 'Run build'
-    inputs:
-      targetType: filePath
-      filePath: './helpers/azure-devops/run-ci-builds.ps1 '
-      arguments: |
-          -TeamFoundationCollectionUri $(System.TeamFoundationCollectionUri) `
-          -AzureDevOpsProjectName $(System.TeamProject) `
-          -AzureDevOpsAccessToken $(System.AccessToken) `
-          -SourceBranch $(Build.SourceBranch) `
-          -SourceVersion $(Build.SourceVersion) `
-          -ToolVersions "$(PYTHON_VERSIONS)" `
-          -DefinitionId $(DEFINITION_ID) 

azure-pipelines/templates/build-job.yml

@@ -1,22 +0,0 @@
-jobs:
-- job: Build_Python
-  timeoutInMinutes: 90
-  pool:
-    name: Azure Pipelines
-    vmImage: $(VmImage)
-  steps:
-  - checkout: self
-    submodules: true
-    
-  - task: PowerShell@2
-    displayName: 'Build Python $(VERSION)'
-    inputs:
-      targetType: filePath
-      filePath: './builders/build-python.ps1'
-      arguments: '-Version $(VERSION) -Platform $(Platform) -Architecture $(Architecture)'
-
-  - task: PublishPipelineArtifact@1
-    displayName: 'Publish Artifact: Python $(VERSION)'
-    inputs:
-      targetPath: '$(Build.ArtifactStagingDirectory)' 
-      artifactName: 'python-$(VERSION)-$(Platform)-$(Architecture)'

azure-pipelines/templates/test-job.yml

@@ -1,80 +0,0 @@
-jobs:
-- job: Test_Python
-  pool: 
-    name: Azure Pipelines
-    vmImage: $(VmImage)
-  steps:
-  - checkout: self
-    submodules: true
-
-  - task: DownloadPipelineArtifact@2
-    inputs:
-      source: 'current'
-      artifact: 'python-$(VERSION)-$(Platform)-$(Architecture)'
-      path: $(Build.BinariesDirectory)
-
-  - task: ExtractFiles@1
-    inputs:
-      archiveFilePatterns: '$(Build.BinariesDirectory)/python-$(VERSION)-$(Platform)-$(Architecture).*'
-      destinationFolder: $(Build.BinariesDirectory)
-      cleanDestinationFolder: false
-
-  - task: PowerShell@2
-    displayName: 'Apply build artifact to the local machines'
-    inputs:
-      TargetType: inline
-      script: |
-        if ($env:PLATFORM -match 'windows') { powershell ./setup.ps1 } else { sh ./setup.sh }
-      workingDirectory: '$(Build.BinariesDirectory)'
-
-  - task: UsePythonVersion@0
-    displayName: 'Use Python $(VERSION)'
-    inputs:
-      versionSpec: '$(VERSION)'
-      architecture: '$(Architecture)'
-
-  - task: PowerShell@2
-    displayName: 'Verbose sysconfig dump'
-    inputs:
-      TargetType: inline
-      script: |
-        Invoke-Expression "python ./sources/python-config-output.py"
-      workingDirectory: '$(Build.SourcesDirectory)/tests'
-    condition: ne(variables['Platform'], 'windows-2016')
-
-  - task: PowerShell@2
-    displayName: 'Verbose python binary links'
-    inputs:
-      TargetType: inline
-      script: |
-        $pythonLocation = which python
-        if ($env:PLATFORM -match 'macos') { otool -L $pythonLocation } else { ldd $pythonLocation }
-      workingDirectory: '$(Build.BinariesDirectory)'
-    condition: ne(variables['Platform'], 'windows-2016')
-
-  - task: PowerShell@2
-    displayName: 'Run tests'
-    inputs:
-      TargetType: inline
-      script: |
-        Install-Module Pester -Force -Scope CurrentUser
-        Import-Module Pester
-        $pesterParams = @{
-          Path="./python-tests.ps1";
-          Parameters=@{
-            Version="$(VERSION)";
-            Platform="$(Platform)";
-          }
-        }
-        Invoke-Pester -Script $pesterParams -OutputFile "test_results.xml" -OutputFormat NUnitXml
-      workingDirectory: '$(Build.SourcesDirectory)/tests'
-
-  - task: PublishTestResults@2
-    displayName: 'Publish test results'
-    inputs:
-      testResultsFiles: '*.xml'
-      testResultsFormat: NUnit
-      searchFolder: 'tests'
-      failTaskOnFailedTests: true
-      testRunTitle: "$(TestRunTitle)"
-    condition: always()

builders/build-python.ps1

@@ -1,6 +1,6 @@
-using module "./builders/win-python-builder.psm1"
-using module "./builders/ubuntu-python-builder.psm1"
-using module "./builders/macos-python-builder.psm1"
+using module "./win-python-builder.psm1"
+using module "./ubuntu-python-builder.psm1"
+using module "./macos-python-builder.psm1"
 
 <#
 .SYNOPSIS
@@ -21,7 +21,7 @@ Required parameter. The platform for which Python will be built.
 #>
 
 param(
-    [Parameter (Mandatory=$true)][Version] $Version,
+    [Parameter (Mandatory=$true)][semver] $Version,
     [Parameter (Mandatory=$true)][string] $Platform,
     [string] $Architecture = "x64"
 )
@@ -29,6 +29,7 @@ param(
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "common-helpers.psm1") -DisableNameChecking
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "nix-helpers.psm1") -DisableNameChecking
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "win-helpers.psm1") -DisableNameChecking
+Import-Module (Join-Path $PSScriptRoot "python-version.psm1") -DisableNameChecking
 
 function Get-PythonBuilder {
     <#
@@ -49,18 +50,17 @@ function Get-PythonBuilder {
 
     #>
 
-    param (
-        [version] $Version,
+    param(
+        [semver] $Version,
         [string] $Architecture,
         [string] $Platform
     )
 
-    $Platform = $Platform.ToLower()  
-    if ($Platform -match 'windows') {
+    if ($Platform -match 'win32') {
         $builder = [WinPythonBuilder]::New($Version, $Architecture, $Platform)
-    } elseif ($Platform -match 'ubuntu') {
+    } elseif ($Platform -match 'linux') {
         $builder = [UbuntuPythonBuilder]::New($Version, $Architecture, $Platform)
-    } elseif ($Platform -match 'macos') {
+    } elseif ($Platform -match 'darwin') {
         $builder = [macOSPythonBuilder]::New($Version, $Architecture, $Platform)
     } else {
         Write-Host "##vso[task.logissue type=error;] Invalid platform: $Platform"
@@ -71,5 +71,5 @@ function Get-PythonBuilder {
 }
 
 ### Create Python builder instance, and build artifact
-$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture  -Platform $Platform 
+$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture -Platform $Platform 
 $Builder.Build()

builders/invoke-workflow.psm1

@@ -0,0 +1,52 @@
+function Invoke-Workflow {
+    param (
+        [string] $Version,
+        [string] $PublishRelease
+    )
+    
+    $payload = @{
+        "ref" = "main"
+        "inputs" = @{
+            "VERSION" = "$Version"
+            "PUBLISH_RELEASES" = "$PublishRelease"
+        }
+    } | ConvertTo-Json
+    $headers = @{
+        Authorization="Bearer $env:TOKEN"
+    }
+    $actionsRepoUri = "$env:GITHUB_API_URL/repos/$env:GITHUB_REPOSITORY/actions"
+    Invoke-RestMethod -uri "$actionsRepoUri/workflows/build-python-packages.yml/dispatches" -method POST -headers $headers -body $payload
+
+    $result = [PSCustomObject]@{
+        Version = $Version
+        Conclusion = "failure"
+        Url = "Not run"
+    }
+    # Triggering workflow and verifying that it has been triggered with retries
+    while (-not $workflowToCheck) {
+        Start-Sleep -seconds 40
+        $workflowRuns = (Invoke-RestMethod "$actionsRepoUri/runs").workflow_runs | Where-Object {$_.status -like "*progress*" -and $_.id -ne $env:GITHUB_RUN_ID}
+        $workflowToCheck = $workflowRuns | Where-Object {
+            (Invoke-RestMethod "$actionsRepoUri/runs/$($_.id)/jobs").jobs.steps.name -like "*$Version"
+        }
+        $retries++
+        if ($retries -gt 10) {
+            Write-Host "Workflow triggered for version '$Version' not found or something went wrong with fetching the workflow status"
+            return $result
+        }
+    }
+    # Waiting for workflow to complete
+    while ($workflowToCheck.status -ne "completed") {
+        Start-Sleep -Seconds 120
+        $workflowToCheck = Invoke-RestMethod "$actionsRepoUri/runs/$($workflowToCheck.id)"
+        Write-Host "Workflow run with Id: $($workflowToCheck.id) for version '$Version' - status '$($workflowToCheck.status)'"
+    }
+    $result.Conclusion = $workflowToCheck.conclusion
+    $result.Url = $workflowToCheck.html_url
+    if ($workflowToCheck.conclusion -ne "success") {
+        Write-Host "Triggered workflow for version '$Version' completed unsuccessfully with result '$($workflowToCheck.conclusion)'. Check the logs: $($workflowToCheck.html_url)"
+        return $result
+    }
+    Write-Host "Triggered workflow for version '$Version' succeeded; Url: $($workflowToCheck.html_url)"
+    return $result
+}

builders/macos-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/nix-python-builder.psm1"
+using module "./nix-python-builder.psm1"
 
 class macOSPythonBuilder : NixPythonBuilder {
     <#
@@ -8,6 +8,10 @@ class macOSPythonBuilder : NixPythonBuilder {
     .DESCRIPTION
     Contains methods that required to build macOS Python artifact from sources. Inherited from base NixPythonBuilder.
 
+    While python.org provides precompiled binaries for macOS, switching to them risks breaking existing customers.
+    If we wanted to start using the official binaries instead of building from source, we should avoid changing previous versions
+    so we remain backwards compatible.
+
     .PARAMETER platform
     The full name of platform for which Python should be built.
 
@@ -17,11 +21,26 @@ class macOSPythonBuilder : NixPythonBuilder {
     #>
 
     macOSPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) { }
 
+    [void] PrepareEnvironment() {
+        <#
+        .SYNOPSIS
+        Prepare system environment by installing dependencies and required packages.
+        #>
+        
+        if ($this.Version -eq "3.7.17") {
+            # We have preinstalled ncurses and readLine on the hoster runners. But we need to install bzip2 for 
+            # setting up an environemnt 
+            # If we get any issues realted to ncurses or readline we can try to run this command
+            # brew install ncurses readline
+            Execute-Command -Command "brew install bzip2"
+        }
+    }
+
     [void] Configure() {
         <#
         .SYNOPSIS
@@ -29,30 +48,149 @@ class macOSPythonBuilder : NixPythonBuilder {
         #>
 
         $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation()
-        $configureString = "./configure --prefix=$pythonBinariesLocation --enable-optimizations --enable-shared --with-lto"
+        $configureString = "./configure"
+        $configureString += " --prefix=$pythonBinariesLocation"
+        $configureString += " --enable-optimizations"
+        $configureString += " --enable-shared"
+        $configureString += " --with-lto"
+
+        ### For Python versions which support it, compile a universal2 (arm64 + x86_64 hybrid) build. The arm64 slice
+        ### will never be used itself by a Github Actions runner but using a universal2 Python is the only way to build
+        ### universal2 C extensions and wheels. This is supported by Python >= 3.10 and was backported to Python >=
+        ### 3.9.1 and >= 3.8.10.
+        ### Disabled, discussion: https://github.com/actions/python-versions/pull/114
+        # if ($this.Version -ge "3.8.10" -and $this.Version -ne "3.8.13" -and $this.Version -ne "3.9.0" ) {
+        #     $configureString += " --enable-universalsdk --with-universal-archs=universal2"
+        # }
 
         ### OS X 10.11, Apple no longer provides header files for the deprecated system version of OpenSSL.
         ### Solution is to install these libraries from a third-party package manager,
         ### and then add the appropriate paths for the header and library files to configure command.
         ### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies)
         if ($this.Version -lt "3.7.0") {
-            $env:LDFLAGS="-L$(brew --prefix openssl)/lib"
-            $env:CFLAGS="-I$(brew --prefix openssl)/include"
+            $env:LDFLAGS = "-L/usr/local/opt/openssl@1.1/lib -L/usr/local/opt/zlib/lib"
+            $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/zlib/include"
         } else {
-            $configureString += " --with-openssl=/usr/local/opt/openssl"
+            $configureString += " --with-openssl=/usr/local/opt/openssl@1.1"
+
+            # For Python 3.7.2 and 3.7.3 we need to provide PATH for zlib to pack it properly. Otherwise the build will fail
+            # with the error: zipimport.ZipImportError: can't decompress data; zlib not available
+            if ($this.Version -eq "3.7.2" -or $this.Version -eq "3.7.3" -or $this.Version -eq "3.7.17") {
+                $env:LDFLAGS = "-L/usr/local/opt/zlib/lib"
+                $env:CFLAGS = "-I/usr/local/opt/zlib/include"
+            }
+
+            if ($this.Version -gt "3.7.12") {
+                $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'"
+	        }
+
+            if ($this.Version -eq "3.7.17") {
+                $env:LDFLAGS += " -L$(brew --prefix bzip2)/lib -L$(brew --prefix readline)/lib -L$(brew --prefix ncurses)/lib"
+                $env:CFLAGS += " -I$(brew --prefix bzip2)/include -I$(brew --prefix readline)/include -I$(brew --prefix ncurses)/include"
+            }
         }
 
+        ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
+        ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
+        if ($this.Version -ge "3.2.0") {
+            $configureString += " --enable-loadable-sqlite-extensions"
+            $env:LDFLAGS += " -L$(brew --prefix sqlite3)/lib"
+            $env:CFLAGS += " -I$(brew --prefix sqlite3)/include"
+            $env:CPPFLAGS += "-I$(brew --prefix sqlite3)/include"
+        }
+
+        Write-Host "The passed configure options are: "
+        Write-Host $configureString
+
         Execute-Command -Command $configureString
     }
 
-    [void] PrepareEnvironment() {
+    [string] GetPkgName() {
         <#
         .SYNOPSIS
-        Prepare system environment by installing dependencies and required packages.
+        Return Python installation Package.
         #>
 
-        ### reinstall header files to Avoid issue with X11 headers on Mojave
-        $pkgName = "/Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg"
-        Execute-Command -Command "sudo installer -pkg $pkgName -target /"
+        $nativeVersion = Convert-Version -version $this.Version
+        $architecture = "-macos11"
+        $extension = ".pkg"
+
+        $pkg = "python-${nativeVersion}${architecture}${extension}"
+
+        return $pkg
+    }
+
+    [uri] GetPkgUri() {
+        <#
+        .SYNOPSIS
+        Get base Python URI and return complete URI for Python installation package.
+        #>
+
+        $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $pkg = $this.GetPkgName()
+
+        $uri = "${base}/${versionName}/${pkg}"
+
+        return $uri
+    }
+
+    [string] DownloadPkg() {
+        <#
+        .SYNOPSIS
+        Download Python installation executable into artifact location.
+        #>
+
+        $pkgUri = $this.GetPkgUri()
+
+        Write-Host "Sources URI: $pkgUri"
+        $pkgLocation = Download-File -Uri $pkgUri -OutputFolder $this.WorkFolderLocation
+        Write-Debug "Done; Package location: $pkgLocation"
+
+        New-Item -Path $this.WorkFolderLocation -Name "build_output.txt"  -ItemType File
+        return $pkgLocation
+    }
+
+    [void] CreateInstallationScriptPkg() {
+        <#
+        .SYNOPSIS
+        Create Python artifact installation script based on specified template.
+        #>
+
+        $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath "macos-pkg-setup-template.sh"
+        $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
+        $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName  -ItemType File
+
+        $variablesToReplace = @{
+            "{{__VERSION_FULL__}}" = $this.Version;
+            "{{__PKG_NAME__}}" = $this.GetPkgName();
+            "{{__ARCH__}}" = $this.Architecture;
+        }
+
+        $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
+        $installationTemplateContent | Out-File -FilePath $installationScriptLocation
+        Write-Debug "Done; Installation script location: $installationScriptLocation)"
+    }
+
+    [void] Build() {
+        <#
+        .SYNOPSIS
+        Generates Python artifact from downloaded Python installation executable.
+        #>
+
+        $PkgVersion = [semver]"3.11.0-beta.1"
+
+        if (($this.Version -ge $PkgVersion) -or ($this.Architecture -eq "arm64")) {
+            Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..."
+            $this.DownloadPkg()
+
+            Write-Host "Create installation script..."
+            $this.CreateInstallationScriptPkg()
+        } else {
+            ([NixPythonBuilder]$this).Build()
+        }
+
+        Write-Host "Archive artifact"
+        $this.ArchiveArtifact()
     }
 }

builders/nix-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/python-builder.psm1"
+using module "./python-builder.psm1"
 
 class NixPythonBuilder : PythonBuilder {
     <#
@@ -33,13 +33,12 @@ class NixPythonBuilder : PythonBuilder {
     [string] $OutputArtifactName
 
     NixPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) {
         $this.InstallationTemplateName = "nix-setup-template.sh"	
         $this.InstallationScriptName = "setup.sh"
-
         $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.tar.gz"
     }
 
@@ -50,8 +49,10 @@ class NixPythonBuilder : PythonBuilder {
         #>
 
         $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $nativeVersion = Convert-Version -version $this.Version
 
-        return "${base}/$($this.Version)/Python-$($this.Version).tgz"
+        return "${base}/${versionName}/Python-${nativeVersion}.tgz"
     }
 
     [string] GetPythonBinary() {
@@ -93,7 +94,12 @@ class NixPythonBuilder : PythonBuilder {
         $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath $this.InstallationTemplateName
 
         $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
-        $installationTemplateContent = $installationTemplateContent -f $this.Version.Major, $this.Version.Minor, $this.Version.Build
+
+        $variablesToReplace = @{
+            "{{__VERSION_FULL__}}" = $this.Version;
+        }
+        $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
+
         $installationTemplateContent | Out-File -FilePath $installationScriptLocation
 
         Write-Debug "Done; Installation script location: $installationScriptLocation)"

builders/python-builder.psm1

@@ -29,7 +29,7 @@ class PythonBuilder {
 
     #>
 
-    [version] $Version
+    [semver] $Version
     [string] $Architecture
     [string] $Platform
     [string] $HostedToolcacheLocation
@@ -38,17 +38,23 @@ class PythonBuilder {
     [string] $ArtifactFolderLocation
     [string] $InstallationTemplatesLocation
 
-    PythonBuilder ([version] $version, [string] $architecture, [string] $platform) {
+    PythonBuilder ([semver] $version, [string] $architecture, [string] $platform) {
+        $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers"
+
+        $artifactDirectory = Join-Path $env:RUNNER_TEMP "artifact"
+        $workDirectory = Join-Path $env:RUNNER_TEMP "work"
+
+        New-Item -Force -Type Directory $artifactDirectory
+        New-Item -Force -Type Directory $workDirectory
+
+        $this.HostedToolcacheLocation = $env:RUNNER_TOOL_CACHE
+        $this.TempFolderLocation = $env:RUNNER_TEMP
+        $this.WorkFolderLocation =  $workDirectory
+        $this.ArtifactFolderLocation = $artifactDirectory
+
         $this.Version = $version
         $this.Architecture = $architecture
         $this.Platform = $platform
-
-        $this.HostedToolcacheLocation = $env:AGENT_TOOLSDIRECTORY
-        $this.TempFolderLocation = $env:BUILD_SOURCESDIRECTORY
-        $this.WorkFolderLocation = $env:BUILD_BINARIESDIRECTORY
-        $this.ArtifactFolderLocation = $env:BUILD_STAGINGDIRECTORY
-
-        $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers"
     }
 
     [uri] GetBaseUri() {
@@ -79,11 +85,21 @@ class PythonBuilder {
         return "$pythonToolcacheLocation/$($this.Version)/$($this.Architecture)"
     }
 
+    [string] GetBaseVersion() {
+        <#
+        .SYNOPSIS
+        Return Major.Minor.Patch version string.
+        #>
+
+        return "$($this.Version.Major).$($this.Version.Minor).$($this.Version.Patch)"
+    }
+
     [void] PreparePythonToolcacheLocation() {
         <#
         .SYNOPSIS
         Prepare system hostedtoolcache folder for new Python version. 
         #>
+        
         $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation()
 
         if (Test-Path $pythonBinariesLocation) {

builders/python-version.psm1

@@ -0,0 +1,45 @@
+function Convert-Label() {
+    <#
+    .SYNOPSIS
+    Convert generic semver label to native Python label.
+    #>
+
+    param(
+        [Parameter(Mandatory)]
+        [string] $label
+    )
+
+    switch ($label) {
+        "alpha" { return "a" }
+        "beta" { return "b" }
+        "rc" { return "rc" }
+        default { throw "Invalid version label '$label'" }
+    }
+}
+
+function Convert-Version {
+    <#
+    .SYNOPSIS
+    Convert generic semver version to native Python version.
+    #>
+
+    param(
+        [Parameter(Mandatory)]
+        [semver] $version,
+        [char] $delimiter = "."
+    )
+
+    $nativeVersion = "{0}.{1}.{2}" -f $version.Major, $version.Minor, $version.Patch
+
+    if ($version.PreReleaseLabel)
+    {
+        $preReleaseLabel = $version.PreReleaseLabel.Split($delimiter)
+        
+        $preReleaseLabelName = Convert-Label -Label $preReleaseLabel[0]
+        $preReleaseLabelVersion = $preReleaseLabel[1]
+
+        $nativeVersion += "${preReleaseLabelName}${preReleaseLabelVersion}"
+    }
+
+    return $nativeVersion
+}

builders/python-versions-runner.ps1

@@ -0,0 +1,31 @@
+<#
+.SYNOPSIS
+Generate Python artifact.
+
+.DESCRIPTION
+Script that triggering and fetching the result of the "Build python package" workflows with provided python versions
+
+.PARAMETER Version
+Required parameter. Python versions to trigger builds for.
+
+.PARAMETER PublishRelease
+Switch parameter. Whether to publish release for built version.
+
+#>
+
+param(
+    [Parameter (Mandatory=$true, HelpMessage="Python version to trigger build for")]
+    [array] $Versions,
+    [Parameter (Mandatory=$false, HelpMessage="Whether to publish release for built version")]
+    [string] $PublishRelease
+)
+
+$summary = $Versions | ForEach-Object -Parallel { 
+    Import-Module "./builders/invoke-workflow.psm1"
+    Invoke-Workflow -Version $_ -PublishRelease $Using:PublishRelease
+}
+Write-Host "Results of triggered workflows:"
+$summary | Out-String
+if ($summary.Conclusion -contains "failure" -or $summary.Conclusion -contains "cancelled") {
+    exit 1
+}

builders/ubuntu-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/nix-python-builder.psm1"
+using module "./nix-python-builder.psm1"
 
 class UbuntuPythonBuilder : NixPythonBuilder {
     <#
@@ -17,7 +17,7 @@ class UbuntuPythonBuilder : NixPythonBuilder {
     #>
 
     UbuntuPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) { }
@@ -32,13 +32,25 @@ class UbuntuPythonBuilder : NixPythonBuilder {
 
         ### To build Python with SO we must pass full path to lib folder to the linker
         $env:LDFLAGS="-Wl,--rpath=${pythonBinariesLocation}/lib"
-        $configureString = "./configure --prefix=$pythonBinariesLocation --enable-shared --enable-optimizations"
+        $configureString = "./configure"
+        $configureString += " --prefix=$pythonBinariesLocation"
+        $configureString += " --enable-shared"
+        $configureString += " --enable-optimizations"
 
+        ### Compile with ucs4 for Python 2.x. On 3.x, ucs4 is enabled by default
         if ($this.Version -lt "3.0.0") {
-            ### Compile with ucs4 for Python 2.x. On 3.x, ucs4 is enabled by default
             $configureString += " --enable-unicode=ucs4"
         }
 
+        ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
+        ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
+        if ($this.Version -ge "3.2.0") {
+            $configureString += " --enable-loadable-sqlite-extensions"
+        }
+
+        Write-Host "The passed configure options are: "
+        Write-Host $configureString
+
         Execute-Command -Command $configureString
     }
 
@@ -59,6 +71,8 @@ class UbuntuPythonBuilder : NixPythonBuilder {
         } else {
             $tkinterInstallString = "sudo apt install -y python-tk tk-dev"
         }
+
+        Execute-Command -Command "sudo apt-get update"
         Execute-Command -Command $tkinterInstallString
 
         ### Install dependent packages
@@ -76,10 +90,7 @@ class UbuntuPythonBuilder : NixPythonBuilder {
             Execute-Command -Command "sudo apt install -y $_"
         }
 
-        if ($this.Platform -ne "ubuntu-1604") {
-            ### On Ubuntu-1804, libgdbm-compat-dev has older modules that are no longer in libgdbm-dev
-            Execute-Command -Command "sudo apt install -y libgdbm-compat-dev"
-        }
-        
+        ### On Ubuntu-1804, libgdbm-compat-dev has older modules that are no longer in libgdbm-dev
+        Execute-Command -Command "sudo apt install -y libgdbm-compat-dev"
     }
 }

builders/win-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/python-builder.psm1"
+using module "./python-builder.psm1"
 
 class WinPythonBuilder : PythonBuilder {
     <#
@@ -27,7 +27,7 @@ class WinPythonBuilder : PythonBuilder {
     [string] $OutputArtifactName
 
     WinPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) {
@@ -72,10 +72,12 @@ class WinPythonBuilder : PythonBuilder {
         #>
 
         $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $nativeVersion = Convert-Version -version $this.Version
         $architecture = $this.GetArchitectureExtension()
         $extension = $this.GetPythonExtension()
 
-        $uri = "${base}/$($this.Version)/python-$($this.Version)${architecture}${extension}"
+        $uri = "${base}/${versionName}/python-${nativeVersion}${architecture}${extension}"
 
         return $uri
     }

config/python-manifest-config.json

@@ -0,0 +1,8 @@
+{
+    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+)",
+    "groups": {
+        "arch": 4,
+        "platform": 2,
+        "platform_version": 3
+    }
+}

helpers/azure-devops/azure-devops-api.ps1

@@ -1,89 +0,0 @@
-class AzureDevOpsApi
-{
-    [string] $BaseUrl
-    [string] $RepoOwner
-    [object] $AuthHeader
-
-    AzureDevOpsApi(
-        [string] $TeamFoundationCollectionUri,
-        [string] $ProjectName,
-        [string] $AccessToken
-    ) {
-        $this.BaseUrl = $this.BuildBaseUrl($TeamFoundationCollectionUri, $ProjectName)
-        $this.AuthHeader = $this.BuildAuth($AccessToken)
-    }
-
-    [object] hidden BuildAuth([string]$AccessToken) {
-        if ([string]::IsNullOrEmpty($AccessToken)) {
-            return $null
-        }
-        return @{
-            Authorization = "Bearer $AccessToken"
-        }
-    }
-
-    [string] hidden BuildBaseUrl([string]$TeamFoundationCollectionUri, [string]$ProjectName) {
-        return "${TeamFoundationCollectionUri}/${ProjectName}/_apis"
-    }
-
-    [object] QueueBuild([string]$ToolVersion, [string]$SourceBranch, [string]$SourceVersion, [UInt32]$DefinitionId){
-        $url = "build/builds"
-        
-        # The content of parameters field should be a json string
-        $buildParameters = @{ VERSION = $ToolVersion } | ConvertTo-Json
-
-        $body = @{
-            definition = @{
-                id = $DefinitionId
-            }
-            sourceBranch = $SourceBranch
-            sourceVersion = $SourceVersion
-            parameters = $buildParameters
-        } | ConvertTo-Json
-
-        return $this.InvokeRestMethod($url, 'POST', $body)
-    }
-
-    [object] GetBuildInfo([UInt32]$BuildId){
-        $url = "build/builds/$BuildId"
-
-        return $this.InvokeRestMethod($url, 'GET', $null)
-    }
-
-    [string] hidden BuildUrl([string]$Url) {
-        return "$($this.BaseUrl)/${Url}/?api-version=5.1"
-    }
-
-    [object] hidden InvokeRestMethod(
-        [string] $Url,
-        [string] $Method,
-        [string] $Body
-    ) {
-        $requestUrl = $this.BuildUrl($Url)
-        $params = @{
-            Method = $Method
-            ContentType = "application/json"
-            Uri = $requestUrl
-            Headers = @{}
-        }
-        if ($this.AuthHeader) {
-            $params.Headers += $this.AuthHeader
-        }
-        if (![string]::IsNullOrEmpty($body)) {
-            $params.Body = $Body
-        }
-
-        return Invoke-RestMethod @params
-    }
-
-}
-
-function Get-AzureDevOpsApi {
-    param (
-        [string] $TeamFoundationCollectionUri,
-        [string] $ProjectName,
-        [string] $AccessToken
-    )
-
-    return [AzureDevOpsApi]::New($TeamFoundationCollectionUri, $ProjectName, $AccessToken)
-}

helpers/azure-devops/build-info.ps1

@@ -1,44 +0,0 @@
-Import-Module (Join-Path $PSScriptRoot "azure-devops-api.ps1")
-
-class BuildInfo
-{
-    [AzureDevOpsApi] $AzureDevOpsApi
-    [String] $Name
-    [UInt32] $Id
-    [String] $Status
-    [String] $Result
-    [String] $Link
-
-    BuildInfo([AzureDevOpsApi] $AzureDevOpsApi, [object] $Build)
-    {
-        $this.AzureDevOpsApi = $AzureDevOpsApi
-        $this.Id = $Build.id
-        $this.Name = $Build.buildNumber
-        $this.Link = $Build._links.web.href
-        $this.Status = $Build.status
-        $this.Result = $Build.result
-    }
-
-    [boolean] IsFinished() {
-        return ($this.Status -eq "completed") -or ($this.Status -eq "cancelling")
-    }
-
-    [boolean] IsSuccess() {
-        return $this.Result -eq "succeeded"
-    }
-
-    [void] UpdateBuildInfo() {
-        $buildInfo = $this.AzureDevOpsApi.GetBuildInfo($this.Id)
-        $this.Status = $buildInfo.status
-        $this.Result = $buildInfo.result
-    }
-}
-
-function Get-BuildInfo {
-    param (
-        [AzureDevOpsApi] $AzureDevOpsApi,
-        [object] $Build
-    )
-
-    return [BuildInfo]::New($AzureDevOpsApi, $Build)
-}

helpers/azure-devops/run-ci-builds.ps1

@@ -1,94 +0,0 @@
-param (
-    [Parameter(Mandatory)] [string] $TeamFoundationCollectionUri,
-    [Parameter(Mandatory)] [string] $AzureDevOpsProjectName,
-    [Parameter(Mandatory)] [string] $AzureDevOpsAccessToken,
-    [Parameter(Mandatory)] [string] $SourceBranch,
-    [Parameter(Mandatory)] [string] $ToolVersions,
-    [Parameter(Mandatory)] [UInt32] $DefinitionId,
-    [string] $SourceVersion
-)
-
-Import-Module (Join-Path $PSScriptRoot "azure-devops-api.ps1")
-Import-Module (Join-Path $PSScriptRoot "build-info.ps1")
-
-function Queue-Builds {
-    param (
-        [Parameter(Mandatory)] [AzureDevOpsApi] $AzureDevOpsApi,
-        [Parameter(Mandatory)] [string] $ToolVersions,
-        [Parameter(Mandatory)] [string] $SourceBranch,
-        [Parameter(Mandatory)] [string] $SourceVersion,
-        [Parameter(Mandatory)] [string] $DefinitionId
-    )
-
-    [BuildInfo[]]$queuedBuilds = @()
-
-    $ToolVersions.Split(',') | ForEach-Object { 
-        $version = $_.Trim()
-        Write-Host "Queue build for $version..."
-        $queuedBuild = $AzureDevOpsApi.QueueBuild($version, $SourceBranch, $SourceVersion, $DefinitionId)
-        $buildInfo = Get-BuildInfo -AzureDevOpsApi $AzureDevOpsApi -Build $queuedBuild
-        Write-Host "Queued build: $($buildInfo.Link)"
-        $queuedBuilds += $buildInfo
-    }
-
-    return $queuedBuilds
-}
-
-function Wait-Builds {
-    param (
-        [Parameter(Mandatory)] [BuildInfo[]] $Builds
-    )
-
-    $timeoutBetweenRefreshSec = 30
-    
-    do {
-        # If build is still running - refresh its status
-        foreach($build in $builds) {
-            if (!$build.IsFinished()) {
-                $build.UpdateBuildInfo()
-                
-                if ($build.IsFinished()) {
-                   Write-Host "The $($build.Name) build was completed: $($build.Link)"
-                }
-            }
-        }
-    
-        $runningBuildsCount = ($builds | Where-Object { !$_.IsFinished() }).Length
-
-        Start-Sleep -Seconds $timeoutBetweenRefreshSec
-    } while($runningBuildsCount -gt 0)
-}
-
-function Make-BuildsOutput {
-    param (
-        [Parameter(Mandatory)] [BuildInfo[]] $Builds
-    )
-
-    Write-Host "Builds info:"
-    $builds | Format-Table -AutoSize -Property Name,Id,Status,Result,Link | Out-String -Width 10000
-
-    # Return exit code based on status of builds
-    $failedBuilds = ($builds | Where-Object { !$_.IsSuccess() })
-    if ($failedBuilds.Length -ne 0) {
-        Write-Host "##vso[task.logissue type=error;]Builds failed"
-        $failedBuilds | ForEach-Object -Process { Write-Host "##vso[task.logissue type=error;]Name: $($_.Name); Link: $($_.Link)" }
-        Write-Host "##vso[task.complete result=Failed]"
-    } else {
-        Write-host "##[section] All builds have been passed successfully"
-    }
-}
-
-$azureDevOpsApi = Get-AzureDevOpsApi -TeamFoundationCollectionUri $TeamFoundationCollectionUri `
-                                     -ProjectName $AzureDevOpsProjectName `
-                                     -AccessToken $AzureDevOpsAccessToken
-
-$queuedBuilds = Queue-Builds -AzureDevOpsApi $azureDevOpsApi `
-                             -ToolVersions $ToolVersions `
-                             -SourceBranch $SourceBranch `
-                             -SourceVersion $SourceVersion `
-                             -DefinitionId $DefinitionId
-
-Write-Host "Waiting results of builds ..."
-Wait-Builds -Builds $queuedBuilds
-
-Make-BuildsOutput -Builds $queuedBuilds

helpers/common-helpers.psm1

@@ -1,80 +0,0 @@
-<#
-.SYNOPSIS
-The execute command and print all output to the logs
-#>
-function Execute-Command {
-    [CmdletBinding()]
-    param(
-        [Parameter(Mandatory=$true)][string] $Command
-    )
-
-    Write-Debug "Execute $Command"
-
-    try {
-        Invoke-Expression $Command | ForEach-Object { Write-Host $_ }
-    }
-    catch {
-        Write-Host "Error happened during command execution: $Command"
-        Write-Host "##vso[task.logissue type=error;] $_"
-    }
-}
-
-<#
-.SYNOPSIS
-Download file from url and return local path to file
-#>
-function Download-File {
-    param(
-        [Parameter(Mandatory=$true)]
-        [Uri]$Uri,
-        [Parameter(Mandatory=$true)]
-        [String]$OutputFolder
-    )
-
-    $targetFilename = [IO.Path]::GetFileName($Uri)
-    $targetFilepath = Join-Path $OutputFolder $targetFilename
-
-    Write-Debug "Download source from $Uri to $OutFile"
-    try {
-        (New-Object System.Net.WebClient).DownloadFile($Uri, $targetFilepath)
-        return $targetFilepath
-    } catch {
-        Write-Host "Error during downloading file from '$Uri'"
-        "$_"
-        exit 1
-    }    
-}
-
-<#
-.SYNOPSIS
-Generate file that contains the list of all files in particular directory
-#>
-function New-ToolStructureDump {
-    param(
-        [Parameter(Mandatory=$true)]
-        [String]$ToolPath,
-        [Parameter(Mandatory=$true)]
-        [String]$OutputFolder
-    )
-
-    $outputFile = Join-Path $OutputFolder "tools_structure.txt"
-
-    $folderContent = Get-ChildItem -Path $ToolPath -Recurse | Sort-Object | Select-Object -Property FullName, Length
-    $folderContent | ForEach-Object {
-        $relativePath = $_.FullName.Replace($ToolPath, "");
-        return "${relativePath}"
-    } | Out-File -FilePath $outputFile
-}
-
-<#
-.SYNOPSIS
-Check if it is macOS / Ubuntu platform
-#>
-function IsNixPlatform {
-    param(
-        [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()]
-        [String]$Platform
-    )
-
-    return ($Platform -match "macos") -or ($Platform -match "ubuntu")
-}

helpers/generate-versions-manifest.ps1

@@ -1,158 +0,0 @@
-<#
-.SYNOPSIS
-Generate versions manifest based on repository releases
-
-.DESCRIPTION
-Versions manifest is needed to find the latest assets for particular version of tool
-.PARAMETER GitHubRepositoryOwner
-Required parameter. The organization which tool repository belongs
-.PARAMETER GitHubRepositoryName
-Optional parameter. The name of tool repository
-.PARAMETER GitHubAccessToken
-Required parameter. PAT Token to overcome GitHub API Rate limit
-.PARAMETER OutputFile
-Required parameter. File "*.json" where generated results will be saved
-.PARAMETER PlatformMapFile
-Optional parameter. Path to the json file with platform map
-Structure example:
-{
-    "macos-1014": [
-        {
-            "platform": "darwin",
-            "platform_version": "10.14"
-        }, ...
-    ], ...
-}
-#>
-
-param (
-    [Parameter(Mandatory)] [string] $GitHubRepositoryOwner,
-    [Parameter(Mandatory)] [string] $GitHubRepositoryName,
-    [Parameter(Mandatory)] [string] $GitHubAccessToken,
-    [Parameter(Mandatory)] [string] $OutputFile,
-    [string] $PlatformMapFile
-)
-
-Import-Module (Join-Path $PSScriptRoot "github/github-api.psm1")
-
-if ($PlatformMapFile -and (Test-Path $PlatformMapFile)) {
-    $PlatformMap = Get-Content $PlatformMapFile -Raw | ConvertFrom-Json -AsHashtable
-} else {
-    $PlatformMap = @{}
-}
-
-function Get-FileNameWithoutExtension {
-    param (
-        [Parameter(Mandatory)][string]$Filename
-    )
-
-    if ($Filename.EndsWith(".tar.gz")) {
-        $Filename = [IO.path]::GetFileNameWithoutExtension($Filename)
-    }
-
-    return [IO.path]::GetFileNameWithoutExtension($Filename)
-}
-
-function New-AssetItem {
-    param (
-        [Parameter(Mandatory)][string]$Filename,
-        [Parameter(Mandatory)][string]$DownloadUrl,
-        [Parameter(Mandatory)][string]$Arch,
-        [Parameter(Mandatory)][string]$Platform,
-        [string]$PlatformVersion
-    )
-    $asset = New-Object PSObject
-
-    $asset | Add-Member -Name "filename" -Value $Filename -MemberType NoteProperty
-    $asset | Add-Member -Name "arch" -Value $Arch -MemberType NoteProperty
-    $asset | Add-Member -Name "platform" -Value $Platform -MemberType NoteProperty
-    if ($PlatformVersion) { $asset | Add-Member -Name "platform_version" -Value $PlatformVersion -MemberType NoteProperty }
-    $asset | Add-Member -Name "download_url" -Value $DownloadUrl -MemberType NoteProperty
-
-    return $asset
-}
-
-function Build-AssetsList {
-    param (
-        [AllowEmptyCollection()]
-        [Parameter(Mandatory)][array]$ReleaseAssets
-    )
-
-    
-    $assets = @()
-    foreach($releaseAsset in $ReleaseAssets) {
-        $filename = Get-FileNameWithoutExtension -Filename $releaseAsset.name
-        $parts = $filename.Split("-")
-        $arch = $parts[-1]
-        $buildPlatform = [string]::Join("-", $parts[2..($parts.Length-2)])
-
-        if ($PlatformMap[$buildPlatform]) {
-            $PlatformMap[$buildPlatform] | ForEach-Object {
-                $assets += New-AssetItem -Filename $releaseAsset.name `
-                                         -DownloadUrl $releaseAsset.browser_download_url `
-                                         -Arch $arch `
-                                         -Platform $_.platform `
-                                         -PlatformVersion $_.platform_version
-            }
-
-        } else {
-            $assets += New-AssetItem -Filename $releaseAsset.name `
-                                     -DownloadUrl $releaseAsset.browser_download_url `
-                                     -Arch $arch `
-                                     -Platform $buildPlatform
-        }
-    }
-
-    return $assets
-}
-
-function Get-VersionFromRelease {
-    param (
-        [Parameter(Mandatory)][object]$Release
-    )
-    # Release name can contain additional information after ':' so filter it
-    [string]$releaseName = $Release.name.Split(':')[0]
-    [Version]$version = $null
-    if (![Version]::TryParse($releaseName, [ref]$version)) {
-        throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )"
-    }
-
-    return $version
-}
-
-function Build-VersionsManifest {
-    param (
-        [Parameter(Mandatory)][array]$Releases
-    )
-
-    $Releases = $Releases | Sort-Object -Property "published_at" -Descending
-
-    $versionsHash = @{}
-    foreach ($release in $Releases) {
-        if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) {
-            continue
-        }
-
-        [Version]$version = Get-VersionFromRelease $release
-        $versionKey = $version.ToString()
-
-        if ($versionsHash.ContainsKey($versionKey)) {
-            continue
-        }
-
-        $versionsHash.Add($versionKey, [PSCustomObject]@{
-            version = $versionKey
-            stable = $true
-            release_url = $release.html_url
-            files = Build-AssetsList $release.assets
-        })
-    }
-
-    # Sort versions by descending
-    return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true }
-}
-
-$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken
-$releases = $gitHubApi.GetGitHubReleases()
-$versionIndex = Build-VersionsManifest $releases
-$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force

helpers/github/create-pull-request.ps1

@@ -1,106 +0,0 @@
-<#
-.SYNOPSIS
-Create commit with all unstaged changes in repository and create pull-request
-
-.PARAMETER RepositoryOwner
-Required parameter. The organization which tool repository belongs
-.PARAMETER RepositoryName
-Optional parameter. The name of tool repository
-.PARAMETER AccessToken
-Required parameter. PAT Token to authorize
-.PARAMETER BranchName
-Required parameter. The name of branch where changes will be pushed
-.PARAMETER CommitMessage
-Required parameter. The commit message to push changes
-.PARAMETER PullRequestTitle
-Required parameter. The title of pull-request
-.PARAMETER PullRequestBody
-Required parameter. The description of pull-request
-#>
-param (
-    [Parameter(Mandatory)] [string] $RepositoryOwner,
-    [Parameter(Mandatory)] [string] $RepositoryName,
-    [Parameter(Mandatory)] [string] $AccessToken,
-    [Parameter(Mandatory)] [string] $BranchName,
-    [Parameter(Mandatory)] [string] $CommitMessage,
-    [Parameter(Mandatory)] [string] $PullRequestTitle,
-    [Parameter(Mandatory)] [string] $PullRequestBody
-)
-
-Import-Module (Join-Path $PSScriptRoot "github-api.psm1")
-Import-Module (Join-Path $PSScriptRoot "git.psm1")
-
-function Update-PullRequest {
-    Param (
-        [Parameter(Mandatory=$true)]
-        [object] $GitHubApi,
-        [Parameter(Mandatory=$true)]
-        [string] $Title,
-        [Parameter(Mandatory=$true)]
-        [string] $Body,
-        [Parameter(Mandatory=$true)]
-        [string] $BranchName,
-        [Parameter(Mandatory=$true)]
-        [object] $PullRequest
-    )
-
-    $updatedPullRequest = $GitHubApi.UpdatePullRequest($Title, $Body, $BranchName, $PullRequest.number)
-
-    if (($updatedPullRequest -eq $null) -or ($updatedPullRequest.html_url -eq $null)) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected error occurs while updating pull request."
-        exit 1
-    }
-    Write-host "##[section] Pull request updated: $($updatedPullRequest.html_url)"
-}
-
-function Create-PullRequest {
-    Param (
-        [Parameter(Mandatory=$true)]
-        [object] $GitHubApi,
-        [Parameter(Mandatory=$true)]
-        [string] $Title,
-        [Parameter(Mandatory=$true)]
-        [string] $Body,
-        [Parameter(Mandatory=$true)]
-        [string] $BranchName
-    )
-
-    $createdPullRequest = $GitHubApi.CreateNewPullRequest($Title, $Body, $BranchName)
-
-    if (($createdPullRequest -eq $null) -or ($createdPullRequest.html_url -eq $null)) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected error occurs while creating pull request."
-        exit 1
-    }
-
-    Write-host "##[section] Pull request created: $($createdPullRequest.html_url)"
-}
-
-Write-Host "Configure local git preferences"
-Git-ConfigureUser -Name "Service account" -Email "no-reply@microsoft.com"
-
-Write-Host "Create branch: $BranchName"
-Git-CreateBranch -Name $BranchName
-    
-Write-Host "Create commit"
-Git-CommitAllChanges -Message $CommitMessage
-
-Write-Host "Push branch: $BranchName"
-Git-PushBranch -Name $BranchName -Force $true
-
-$gitHubApi = Get-GitHubApi -AccountName $RepositoryOwner -ProjectName $RepositoryName -AccessToken $AccessToken
-$pullRequest = $gitHubApi.GetPullRequest($BranchName, $RepositoryOwner)
-
-if ($pullRequest.Count -gt 0) {
-    Write-Host "Update pull request"
-    Update-PullRequest -GitHubApi $gitHubApi `
-                       -Title $PullRequestTitle `
-                       -Body $PullRequestBody `
-                       -BranchName $BranchName `
-                       -PullRequest $pullRequest[0]
-} else {
-    Write-Host "Create pull request"
-    Create-PullRequest -GitHubApi $gitHubApi `
-                       -Title $PullRequestTitle `
-                       -Body $PullRequestBody `
-                       -BranchName $BranchName
-}

helpers/github/git.psm1

@@ -1,81 +0,0 @@
-<#
-.SYNOPSIS
-Configure git credentials to use with commits
-#>
-function Git-ConfigureUser {
-    Param (
-        [Parameter(Mandatory=$true)]
-        [string] $Name,
-        [Parameter(Mandatory=$true)]
-        [string] $Email
-    )
-
-    git config --global user.name $Name | Out-Host
-    git config --global user.email $Email | Out-Host
-
-    if ($LASTEXITCODE -ne 0) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while configuring git preferences."
-        exit 1
-    }
-}
-
-<#
-.SYNOPSIS
-Create new branch
-#>
-function Git-CreateBranch {
-    Param (
-        [Parameter(Mandatory=$true)]
-        [string] $Name
-    )
-
-    git checkout -b $Name | Out-Host
-
-    if ($LASTEXITCODE -ne 0) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while creating new branch: $Name."
-        exit 1
-    }
-}
-
-<#
-.SYNOPSIS
-Commit all staged and unstaged changes
-#>
-function Git-CommitAllChanges {
-    Param (
-        [Parameter(Mandatory=$true)]
-        [string] $Message
-    )
-
-    git add -A | Out-Host
-    git commit -m "$Message" | Out-Host
-
-    if ($LASTEXITCODE -ne 0) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while commiting changes."
-        exit 1
-    }
-}
-
-<#
-.SYNOPSIS
-Push branch to remote repository
-#>
-function Git-PushBranch {
-    Param (
-        [Parameter(Mandatory=$true)]
-        [string] $Name,
-        [Parameter(Mandatory=$true)]
-        [boolean] $Force
-    )
-
-    if ($Force) {
-        git push --set-upstream origin $Name --force | Out-Host
-    } else {
-        git push --set-upstream origin $Name | Out-Host
-    }
-    
-    if ($LASTEXITCODE -ne 0) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while pushing changes."
-        exit 1
-    }
-}

helpers/github/github-api.psm1

@@ -1,109 +0,0 @@
-<#
-.SYNOPSIS
-The module that contains a bunch of methods to interact with GitHub API V3
-#>
-class GitHubApi
-{
-    [string] $BaseUrl
-    [string] $RepoOwner
-    [object] $AuthHeader
-
-    GitHubApi(
-        [string] $AccountName,
-        [string] $ProjectName,
-        [string] $AccessToken
-    ) {
-        $this.BaseUrl = $this.BuildBaseUrl($AccountName, $ProjectName)
-        $this.AuthHeader = $this.BuildAuth($AccessToken)
-    }
-
-    [object] hidden BuildAuth([string]$AccessToken) {
-        if ([string]::IsNullOrEmpty($AccessToken)) {
-            return $null
-        }
-        $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("'':${AccessToken}"))
-        return @{
-            Authorization = "Basic ${base64AuthInfo}"
-        }
-    }
-
-    [string] hidden BuildBaseUrl([string]$RepositoryOwner, [string]$RepositoryName) {
-        return "https://api.github.com/repos/$RepositoryOwner/$RepositoryName"
-    }
-
-    [object] CreateNewPullRequest([string]$Title, [string]$Body, [string]$BranchName){
-        $requestBody = @{
-            title = $Title
-            body = $Body
-            head = $BranchName
-            base = "master"
-        } | ConvertTo-Json
-
-        $url = "pulls"
-        return $this.InvokeRestMethod($url, 'Post', $null, $requestBody)
-    }
-
-    [object] GetPullRequest([string]$BranchName, [string]$RepositoryOwner){
-        $url = "pulls"
-        return $this.InvokeRestMethod($url, 'GET', "head=${RepositoryOwner}:$BranchName&base=master", $null)
-    }
-
-    [object] UpdatePullRequest([string]$Title, [string]$Body, [string]$BranchName, [string]$PullRequestNumber){
-        $requestBody = @{
-            title = $Title
-            body = $Body
-            head = $BranchName
-            base = "master"
-        } | ConvertTo-Json
-
-        $url = "pulls/$PullRequestNumber"
-        return $this.InvokeRestMethod($url, 'Post', $null, $requestBody)
-    }
-
-    [object] GetGitHubReleases(){
-        $url = "releases"
-        return $this.InvokeRestMethod($url, 'GET', $null, $null)
-    }
-
-    [string] hidden BuildUrl([string]$Url, [string]$RequestParams) {
-        if ([string]::IsNullOrEmpty($RequestParams)) {
-            return "$($this.BaseUrl)/$($Url)"
-        } else {
-            return "$($this.BaseUrl)/$($Url)?$($RequestParams)"
-        }
-    }
-
-    [object] hidden InvokeRestMethod(
-        [string] $Url,
-        [string] $Method,
-        [string] $RequestParams,
-        [string] $Body
-    ) {
-        $requestUrl = $this.BuildUrl($Url, $RequestParams)
-        $params = @{
-            Method = $Method
-            ContentType = "application/json"
-            Uri = $requestUrl
-            Headers = @{}
-        }
-        if ($this.AuthHeader) {
-            $params.Headers += $this.AuthHeader
-        }
-        if (![string]::IsNullOrEmpty($Body)) {
-            $params.Body = $Body
-        }
-
-        return Invoke-RestMethod @params
-    }
-
-}
-
-function Get-GitHubApi {
-    param (
-        [string] $AccountName,
-        [string] $ProjectName,
-        [string] $AccessToken
-    )
-
-    return [GitHubApi]::New($AccountName, $ProjectName, $AccessToken)
-}

helpers/nix-helpers.psm1

@@ -1,50 +0,0 @@
-<#
-.SYNOPSIS
-Pack folder to *.zip format
-#>
-function Pack-Zip {
-    param(
-        [Parameter(Mandatory=$true)]
-        [String]$PathToArchive,
-        [Parameter(Mandatory=$true)]
-        [String]$ToolZipFile
-    )
-
-    Write-Debug "Pack $PathToArchive to $ToolZipFile"
-    Push-Location -Path $PathToArchive
-    zip -q -r $ToolZipFile * | Out-Null
-    Pop-Location
-}
-
-<#
-.SYNOPSIS
-Unpack *.tar file
-#>
-function Extract-TarArchive {
-    param(
-        [Parameter(Mandatory=$true)]
-        [String]$ArchivePath,
-        [Parameter(Mandatory=$true)]
-        [String]$OutputDirectory
-    )
-
-    Write-Debug "tar -C $OutputDirectory -xzf $ArchivePath --strip 1"
-    tar -C $OutputDirectory -xzf $ArchivePath --strip 1
-}
-
-function Create-TarArchive {
-    param(
-        [Parameter(Mandatory=$true)]
-        [String]$SourceFolder,
-        [Parameter(Mandatory=$true)]
-        [String]$ArchivePath,
-        [string]$CompressionType = "gz"
-    )
-
-    $CompressionTypeArgument = If ([string]::IsNullOrWhiteSpace($CompressionType)) { "" } else { "--${CompressionType}" }
-
-    Push-Location $SourceFolder
-    Write-Debug "tar -c $CompressionTypeArgument -f $ArchivePath ."
-    tar -c $CompressionTypeArgument -f $ArchivePath .
-    Pop-Location
-}

helpers/pester-extensions.psm1

@@ -1,33 +0,0 @@
-<#
-.SYNOPSIS
-Pester extension that allows to run command and validate exit code
-.EXAMPLE
-"python file.py" | Should -ReturnZeroExitCode
-#>
-function ShouldReturnZeroExitCode {
-    Param(
-        [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
-        [String]$ActualValue,
-        [switch]$Negate
-    )
-
-    Write-Host "Run command '${ActualValue}'"
-    Invoke-Expression -Command $ActualValue | ForEach-Object { Write-Host $_ }
-    $actualExitCode = $LASTEXITCODE
-
-    [bool]$succeeded = $actualExitCode -eq 0
-    if ($Negate) { $succeeded = -not $succeeded }
-
-    if (-not $succeeded)
-    {
-        $failureMessage = "Command '${ActualValue}' has finished with exit code ${actualExitCode}"
-    }
-
-    return New-Object PSObject -Property @{
-        Succeeded      = $succeeded
-        FailureMessage = $failureMessage
-    }
-}
-
-Add-AssertionOperator -Name ReturnZeroExitCode `
-                    -Test  $function:ShouldReturnZeroExitCode

helpers/win-helpers.psm1

@@ -1,18 +0,0 @@
-function Create-SevenZipArchive {
-    param(
-        [Parameter(Mandatory=$true)]
-        [String]$SourceFolder,
-        [Parameter(Mandatory=$true)]
-        [String]$ArchivePath,
-        [String]$ArchiveType = "zip",
-        [String]$CompressionLevel = 5
-    )
-
-    $ArchiveTypeArgument = "-t${ArchiveType}"
-    $CompressionLevelArgument = "-mx=${CompressionLevel}"
-    
-    Push-Location $SourceFolder
-    Write-Debug "7z a $ArchiveTypeArgument $CompressionLevelArgument $ArchivePath @$SourceFolder"
-    7z a $ArchiveTypeArgument $CompressionLevelArgument $ArchivePath $SourceFolder\*
-    Pop-Location
-}

installers/macos-pkg-setup-template.sh

@@ -0,0 +1,79 @@
+set -e
+
+PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
+PYTHON_PKG_NAME="{{__PKG_NAME__}}"
+ARCH="{{__ARCH__}}"
+MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
+MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
+
+PYTHON_MAJOR=python$MAJOR_VERSION
+PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION
+PYTHON_MAJOR_MINOR=python$MAJOR_VERSION$MINOR_VERSION
+
+if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then
+    # No AGENT_TOOLSDIRECTORY on GitHub images
+    TOOLCACHE_ROOT=$RUNNER_TOOL_CACHE
+else
+    TOOLCACHE_ROOT=$AGENT_TOOLSDIRECTORY
+fi
+
+PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
+PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
+PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
+PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
+
+echo "Check if Python hostedtoolcache folder exist..."
+if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
+    echo "Creating Python hostedtoolcache folder..."
+    mkdir -p $PYTHON_TOOLCACHE_PATH
+else
+    # remove ALL other directories for same major.minor python versions
+    find $PYTHON_TOOLCACHE_PATH -name "${MAJOR_VERSION}.${MINOR_VERSION}.*"|while read python_version;do
+        python_version_arch="$python_version/$ARCH"
+        if [ -e "$python_version_arch" ];then
+            echo "Deleting Python $python_version_arch"
+            rm -rf "$python_version_arch"
+        fi
+    done
+fi
+
+echo "Install Python binaries from prebuilt package"
+sudo installer -pkg $PYTHON_PKG_NAME -target /
+
+echo "Create hostedtoolcach symlinks (Required for the backward compatibility)"
+echo "Create Python $PYTHON_FULL_VERSION folder"
+mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+
+ln -s "${PYTHON_FRAMEWORK_PATH}/bin" bin
+ln -s "${PYTHON_FRAMEWORK_PATH}/include" include
+ln -s "${PYTHON_FRAMEWORK_PATH}/share" share
+ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib
+
+echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
+ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
+
+cd bin/
+
+# This symlink already exists if Python version with the same major.minor version is installed,
+# since we do not remove the framework folder
+if [ ! -f $PYTHON_MAJOR_MINOR ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
+fi
+
+if [ ! -f python ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR python
+fi
+
+chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
+
+echo "Upgrading pip..."
+./python -m ensurepip
+./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location --root-user-action=ignore
+
+echo "Install OpenSSL certificates"
+sh -e "${PYTHON_APPLICATION_PATH}/Install Certificates.command"
+
+echo "Create complete file"
+touch $PYTHON_TOOLCACHE_VERSION_PATH/${ARCH}.complete

installers/map_python_platforms.json

@@ -1,29 +0,0 @@
-{
-    "macos-1014": [
-        {
-            "platform": "darwin",
-            "platform_version": "10.14"
-        },
-        {
-            "platform": "darwin",
-            "platform_version": "10.15"
-        }
-    ],
-    "ubuntu-1604": [
-        {
-            "platform": "linux",
-            "platform_version": "16.04"
-        }
-    ],
-    "ubuntu-1804": [
-        {
-            "platform": "linux",
-            "platform_version": "18.04"
-        }
-    ],
-    "windows-2016": [
-        {
-            "platform": "win32"
-        }
-    ]
-}

installers/nix-setup-template.sh

@@ -1,15 +1,21 @@
 set -e
 
-MAJOR_VERSION={0}
-MINOR_VERSION={1}
-BUILD_VERSION={2}
+PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
+MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
+MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
 
 PYTHON_MAJOR=python$MAJOR_VERSION
 PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION
 PYTHON_MAJORMINOR=python$MAJOR_VERSION$MINOR_VERSION
-PYTHON_FULL_VERSION=$MAJOR_VERSION.$MINOR_VERSION.$BUILD_VERSION
 
-PYTHON_TOOLCACHE_PATH=$AGENT_TOOLSDIRECTORY/Python
+if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then
+    # No AGENT_TOOLSDIRECTORY on GitHub images
+    TOOLCACHE_ROOT=$RUNNER_TOOL_CACHE
+else
+    TOOLCACHE_ROOT=$AGENT_TOOLSDIRECTORY
+fi
+
+PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
 PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
 PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64
 
@@ -31,7 +37,7 @@ rm $PYTHON_TOOLCACHE_VERSION_ARCH_PATH/setup.sh
 
 cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
 
-echo "Create additional symlinks (Required for UsePythonVersion VSTS task)"
+echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
 ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
 
 cd bin/
@@ -42,9 +48,9 @@ fi
 
 chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR python
 
-echo "Upgrading PIP..."
+echo "Upgrading pip..."
 ./python -m ensurepip
-./python -m pip install --ignore-installed pip
+./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location --root-user-action=ignore
 
 echo "Create complete file"
-touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete
+touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete

installers/win-setup-template.ps1

@@ -1,10 +1,9 @@
 [String] $Architecture = "{{__ARCHITECTURE__}}"
-[Version] $Version = "{{__VERSION__}}"
+[String] $Version = "{{__VERSION__}}"
 [String] $PythonExecName = "{{__PYTHON_EXEC_NAME__}}"
 
 function Get-RegistryVersionFilter {
-    param
-    (
+    param(
         [Parameter(Mandatory)][String] $Architecture,
         [Parameter(Mandatory)][Int32] $MajorVersion,
         [Parameter(Mandatory)][Int32] $MinorVersion
@@ -12,20 +11,15 @@ function Get-RegistryVersionFilter {
 
     $archFilter = if ($Architecture -eq 'x86') { "32-bit" } else { "64-bit" }
     ### Python 2.7 x86 have no architecture postfix
-    if (($Architecture -eq "x86") -and ($MajorVersion -eq 2))
-    {
+    if (($Architecture -eq "x86") -and ($MajorVersion -eq 2)) {
         "Python $MajorVersion.$MinorVersion.\d+$"
-    }
-    else
-    {
+    } else {
         "Python $MajorVersion.$MinorVersion.*($archFilter)"
     }
 }
 
-function Remove-RegistryEntries
-{
-    param
-    (
+function Remove-RegistryEntries {
+    param(
         [Parameter(Mandatory)][String] $Architecture,
         [Parameter(Mandatory)][Int32] $MajorVersion,
         [Parameter(Mandatory)][Int32] $MinorVersion
@@ -34,18 +28,20 @@ function Remove-RegistryEntries
     $versionFilter = Get-RegistryVersionFilter -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
 
     $regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products"
-    $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName
-    foreach ($key in $regKeys)
-    {
-        if ($key.getValue("DisplayName") -match $versionFilter)
-        {
-            Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose
+    if (Test-Path -Path Registry::$regPath) {
+        $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName
+        foreach ($key in $regKeys) {
+            if ($key.getValue("DisplayName") -match $versionFilter) {
+                Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose
+            }
         }
     }
 
     $regPath = "HKEY_CLASSES_ROOT\Installer\Products"
-    Get-ChildItem -Path Registry::$regPath | Where-Object { $_.GetValue("ProductName") -match $versionFilter } | ForEach-Object {
-        Remove-Item Registry::$_ -Recurse -Force -Verbose
+    if (Test-Path -Path Registry::$regPath) {
+        Get-ChildItem -Path Registry::$regPath | Where-Object { $_.GetValue("ProductName") -match $versionFilter } | ForEach-Object {
+            Remove-Item Registry::$_ -Recurse -Force -Verbose
+        }
     }
 
     $uninstallRegistrySections = @(
@@ -63,58 +59,60 @@ function Remove-RegistryEntries
 }
 
 function Get-ExecParams {
-    param
-    (
+    param(
         [Parameter(Mandatory)][Boolean] $IsMSI,
         [Parameter(Mandatory)][String] $PythonArchPath
     )
 
-    if ($IsMSI) 
-    {
+    if ($IsMSI) {
         "TARGETDIR=$PythonArchPath ALLUSERS=1"
-    } 
-    else 
-    {
+    } else {
         "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1"
     }
 }
 
 $ToolcacheRoot = $env:AGENT_TOOLSDIRECTORY
-if ([string]::IsNullOrEmpty($ToolcacheRoot))
-{
+if ([string]::IsNullOrEmpty($ToolcacheRoot)) {
     # GitHub images don't have `AGENT_TOOLSDIRECTORY` variable
     $ToolcacheRoot = $env:RUNNER_TOOL_CACHE
 }
 $PythonToolcachePath = Join-Path -Path $ToolcacheRoot -ChildPath "Python"
-$PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version.ToString()
+$PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version
 $PythonArchPath = Join-Path -Path $PythonVersionPath -ChildPath $Architecture
 
 $IsMSI = $PythonExecName -match "msi"
 
-$MajorVersion = $Version.Major
-$MinorVersion = $Version.Minor
+$MajorVersion = $Version.Split('.')[0]
+$MinorVersion = $Version.Split('.')[1]
 
 Write-Host "Check if Python hostedtoolcache folder exist..."
-if (-Not (Test-Path $PythonToolcachePath)) 
-{
+if (-Not (Test-Path $PythonToolcachePath)) {
     Write-Host "Create Python toolcache folder"
     New-Item -ItemType Directory -Path $PythonToolcachePath | Out-Null
 }
 
 Write-Host "Check if current Python version is installed..."
-$InstalledVersion = Get-ChildItem -Path $PythonToolcachePath -Filter "$MajorVersion.$MinorVersion.*"
+$InstalledVersions = Get-Item "$PythonToolcachePath\$MajorVersion.$MinorVersion.*\$Architecture"
 
-Write-Host "Remove registry entries for Python ${MajorVersion}.${MinorVersion}(${Architecture})..."
-Remove-RegistryEntries -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
+if ($null -ne $InstalledVersions) {
+    Write-Host "Python$MajorVersion.$MinorVersion ($Architecture) was found in $PythonToolcachePath..."
 
-if (($null -ne $InstalledVersion) -and (Test-Path -Path $InstalledVersion.FullName)) {
-    Write-Host "Python$MajorVersion.$MinorVersion was found in $PythonToolcachePath"
-    Write-Host "Deleting $($InstalledVersion.FullName)..."
-    Remove-Item -Path $InstalledVersion.FullName -Recurse -Force
+    foreach ($InstalledVersion in $InstalledVersions) {
+        if (Test-Path -Path $InstalledVersion) {
+            Write-Host "Deleting $InstalledVersion..."
+            Remove-Item -Path $InstalledVersion -Recurse -Force
+            if (Test-Path -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete") {
+                Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose
+            }
+        }
+    }
 } else {
     Write-Host "No Python$MajorVersion.$MinorVersion.* found"
 }
 
+Write-Host "Remove registry entries for Python ${MajorVersion}.${MinorVersion}(${Architecture})..."
+Remove-RegistryEntries -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
+
 Write-Host "Create Python $Version folder in $PythonToolcachePath"
 New-Item -ItemType Directory -Path $PythonArchPath -Force | Out-Null
 
@@ -125,14 +123,18 @@ Write-Host "Install Python $Version in $PythonToolcachePath..."
 $ExecParams = Get-ExecParams -IsMSI $IsMSI -PythonArchPath $PythonArchPath
 
 cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet"
-if ($LASTEXITCODE -ne 0)
-{
+if ($LASTEXITCODE -ne 0) {
     Throw "Error happened during Python installation"
 }
 
+Write-Host "Create `python3` symlink"
+if ($MajorVersion -ne "2") {
+    New-Item -Path "$PythonArchPath\python3.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python.exe"
+}
+
 Write-Host "Install and upgrade Pip"
 $PythonExePath = Join-Path -Path $PythonArchPath -ChildPath "python.exe"
-cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip"
+cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip --no-warn-script-location --root-user-action=ignore"
 
 Write-Host "Create complete file"
 New-Item -ItemType File -Path $PythonVersionPath -Name "$Architecture.complete" | Out-Null

tests/ManifestConfig.Tests.ps1

@@ -0,0 +1,42 @@
+Import-Module (Join-Path $PSScriptRoot "../helpers/packages-generation/manifest-utils.psm1")
+
+$ConfigurationFile = Join-Path $PSScriptRoot "../config/python-manifest-config.json"
+$Configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
+
+$stableTestCases = @(
+    @{ ReleaseName = "python-3.8.3-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
+) | ForEach-Object { $_.Configuration = $Configuration; $_ }
+
+$unstableTestCases = @(
+    @{ ReleaseName = "python-3.9.0-alpha.2-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-beta.2-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-rc.4-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-beta.2-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-beta.2-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
+) | ForEach-Object { $_.Configuration = $Configuration; $_ }
+
+Describe "Python manifest config" {
+    Context "Stable versions" {
+        It "<ReleaseName>" -TestCases $stableTestCases {
+            $Release = @{ name = $ReleaseName }
+            $asset = New-AssetItem -ReleaseAsset $Release -Configuration $Configuration
+            $asset.platform | Should -Be $ExpectedResult.platform
+            $asset.platform_version | Should -Be $ExpectedResult.platform_version
+            $asset.arch | Should -Be $ExpectedResult.arch
+        }
+    }
+
+    Context "Prerelease versions" {
+        It "<ReleaseName>" -TestCases $unstableTestCases {
+            $Release = @{ name = $ReleaseName }
+            $asset = New-AssetItem -ReleaseAsset $Release -Configuration $Configuration
+            $asset.platform | Should -Be $ExpectedResult.platform
+            $asset.platform_version | Should -Be $ExpectedResult.platform_version
+            $asset.arch | Should -Be $ExpectedResult.arch
+        }
+    }
+}

tests/clean-toolcache.ps1

@@ -1,11 +0,0 @@
-if ($env:PLATFORM -match 'windows') {
-  $PythonFilter = "Name like '%Python%'"
-  Get-WmiObject Win32_Product -Filter $PythonFilter | Foreach-Object { 
-      Write-Host "Uninstalling $($_.Name) ..."
-      $_.Uninstall() | Out-Null 
-  }
-}
-
-$PythonToolcachePath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "Python"
-Write-Host "Removing Python toolcache directory ..."
-Remove-Item -Path $PythonToolcachePath -Recurse -Force

tests/python-tests.ps1

@@ -1,31 +1,41 @@
 param (
-    [Version] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    [semver] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
     $Version,
-    [String] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
-    $Platform
+    [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    $Platform,
+    [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    $Architecture
 )
 
 Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
 Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1")
+Import-Module (Join-Path $PSScriptRoot "../builders/python-version.psm1")
 
-function Analyze-MissingModules([string] $buildOutputLocation) {
-    $searchStringStart = "Failed to build these modules:"
-    $searchStringEnd = "running build_scripts"
-    $pattern = "$searchStringStart(.*?)$searchStringEnd"
+BeforeAll {
+    function Analyze-MissingModules([string] $buildOutputLocation) {
+        $searchStringStart = "Failed to build these modules:"
+        $searchStringEnd = "running build_scripts"
+        $pattern = "$searchStringStart(.*?)$searchStringEnd"
 
-    $buildContent = Get-Content -Path $buildOutputLocation
-    $splitBuiltOutput = $buildContent -split "\n";
+        $buildContent = Get-Content -Path $buildOutputLocation
+        $splitBuiltOutput = $buildContent -split "\n";
 
-    ### Search for missing modules that are displayed between the search strings
-    $regexMatch = [regex]::match($SplitBuiltOutput, $Pattern)
-    if ($regexMatch.Success)
-    {
-        Write-Host "Failed missing modules:"
-        Write-Host $regexMatch.Groups[1].Value
-        return 1
+        ### Search for missing modules that are displayed between the search strings
+        $regexMatch = [regex]::match($SplitBuiltOutput, $Pattern)
+        if ($regexMatch.Success)
+        {
+            $module = $regexMatch.Groups[1].Value.Trim()
+            Write-Host "Failed missing modules:"
+            Write-Host $module
+            if ( ($module -eq "_tkinter") -and ( [semver]"$($Version.Major).$($Version.Minor)" -ge [semver]"3.10" -and $Version.PreReleaseLabel ) ) {
+                Write-Host "$module $Version ignored"
+            } else {
+                return 1
+            }
+        }
+
+        return 0
     }
-
-    return 0
 }
 
 Describe "Tests" {
@@ -33,7 +43,7 @@ Describe "Tests" {
         "python --version" | Should -ReturnZeroExitCode
         $pythonLocation = (Get-Command "python").Path
         $pythonLocation | Should -Not -BeNullOrEmpty
-        $expectedPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "Python"
+        $expectedPath = Join-Path -Path $env:RUNNER_TOOL_CACHE -ChildPath "Python"
         $pythonLocation.startsWith($expectedPath) | Should -BeTrue
     }
 
@@ -41,10 +51,28 @@ Describe "Tests" {
         "python ./sources/simple-test.py" | Should -ReturnZeroExitCode
     }
 
+    # linux has no display name and no $DISPLAY environment variable - skip tk test
+    # if (-not (($Platform -match "ubuntu") -or ($Platform -match "linux"))) {
+    #     It "Check if tcl/tk has the same headed and library versions" {
+	#     "python ./sources/tcltk.py" | Should -ReturnZeroExitCode
+    #     }
+    # }
+
+    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($Architecture -ne "arm64"))) {
+        It "Check if sqlite3 module is installed" {
+            "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
+        }
+    }
+
+    It "Run pip" {
+        "pip install requests" | Should -ReturnZeroExitCode
+        "pip uninstall requests -y" | Should -ReturnZeroExitCode
+    }
+
     if (IsNixPlatform $Platform) {
 
         It "Check for failed modules in build_output" {
-            $buildOutputLocation = Join-Path $env:BUILD_BINARIESDIRECTORY "build_output.txt"
+            $buildOutputLocation = Join-Path $env:RUNNER_TEMP "build_output.txt"
             Analyze-MissingModules $buildOutputLocation | Should -Be 0
         }
 
@@ -53,7 +81,8 @@ Describe "Tests" {
         }
 
         It "Check if python configuration is correct" {
-            "python ./sources/python-config-test.py" | Should -ReturnZeroExitCode
+            $nativeVersion = Convert-Version -version $Version
+            "python ./sources/python-config-test.py $Version $nativeVersion $Architecture" | Should -ReturnZeroExitCode
         }
 
         It "Check if shared libraries are linked correctly" {
@@ -62,11 +91,16 @@ Describe "Tests" {
     }
 
     # Pyinstaller 3.5 does not support Python 3.8.0. Check issue https://github.com/pyinstaller/pyinstaller/issues/4311
-    if ($Version -lt "3.8.0") {
+    if ($Version -lt "3.8.0" -and $Version.Major -ne "2") {
         It "Validate Pyinstaller" {
             "pip install pyinstaller" | Should -ReturnZeroExitCode
             "pyinstaller --onefile ./sources/simple-test.py" | Should -ReturnZeroExitCode
-            "./dist/simple-test" | Should -ReturnZeroExitCode
+            $distPath = [IO.Path]::Combine($pwd, "dist", "simple-test")
+            "$distPath" | Should -ReturnZeroExitCode
         }
     }
-}
+
+    It "Check urlopen with HTTPS works" {
+        "python ./sources/python-urlopen-https.py" | Should -ReturnZeroExitCode
+    }
+}

tests/sources/python-config-output.py

@@ -1,6 +1,4 @@
-import distutils.sysconfig
 import sysconfig
 
 from pprint import pprint
 pprint(sysconfig.get_config_vars())
-pprint(distutils.sysconfig.get_config_vars())

tests/sources/python-config-test.py

@@ -1,4 +1,3 @@
-import distutils.sysconfig
 import sysconfig
 import sys
 import platform
@@ -6,7 +5,15 @@ import os
 
 # Define variables
 os_type = platform.system()
-version = sys.version.split(" ")[0]
+version = sys.argv[1]
+nativeVersion = sys.argv[2]
+architecture = sys.argv[3]
+
+versions=version.split(".")
+version_major=int(versions[0])
+version_minor=int(versions[1])
+
+pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (architecture == "arm64"))
 
 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
@@ -17,7 +24,11 @@ have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE")
 ### Define expected variables
 if os_type == 'Linux': expected_ld_library_extension = 'so'
 if os_type == 'Darwin': expected_ld_library_extension = 'dylib'
-expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version)
+
+if pkg_installer:
+    expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib'
+else:
+    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/x64/lib'
 
 # Check modules
 ### Validate libraries path
@@ -28,7 +39,7 @@ if lib_dir_path != expected_lib_dir_path:
 ### Validate shared libraries
 if is_shared:
     print('%s was built with shared extensions' % ld_library_name)
-    
+
     ### Validate libpython extension
     ld_library_extension = ld_library_name.split('.')[-1]
     if ld_library_extension != expected_ld_library_extension:
@@ -36,12 +47,13 @@ if is_shared:
         exit(1)
 else:
     print('%s was built without shared extensions' % ld_library_name)
-    exit(1)
+    if not pkg_installer:
+        exit(1)
 
 ### Validate macOS
 if os_type == 'Darwin':
     ### Validate openssl links
-    if version < "3.7.0":
+    if version_major == 3 and version_minor < 7:
         expected_ldflags = '-L/usr/local/opt/openssl@1.1/lib'
         ldflags = sysconfig.get_config_var('LDFLAGS')
 
@@ -49,20 +61,23 @@ if os_type == 'Darwin':
             print('Invalid ldflags: %s; Expected: %s' % (ldflags, expected_ldflags))
             exit(1)
     else:
-        expected_openssl_includes = '-I/usr/local/opt/openssl/include'
-        expected_openssl_ldflags ='-L/usr/local/opt/openssl/lib'
-        
+        expected_openssl_includes = '-I/usr/local/opt/openssl@1.1/include'
+        expected_openssl_ldflags ='-L/usr/local/opt/openssl@1.1/lib'
+
         openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES')
         openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS')
 
         if openssl_includes != expected_openssl_includes:
             print('Invalid openssl_includes: %s; Expected: %s' % (openssl_includes, expected_openssl_includes))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
         if openssl_ldflags != expected_openssl_ldflags:
             print('Invalid openssl_ldflags: %s; Expected: %s' % (openssl_ldflags, expected_openssl_ldflags))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
 
 ### Validate libreadline
-if not have_libreadline:
-    print('Missing libreadline')
-    exit(1)
+if sys.version_info < (3, 12):
+    if not have_libreadline:
+        print('Missing libreadline')
+        exit(1)

tests/sources/python-modules.py

@@ -139,6 +139,7 @@ standard_library = [
     'sre_constants',
     'sre_parse',
     'ssl',
+    '_ssl',
     'stat',
     'string',
     'stringprep',
@@ -251,6 +252,49 @@ if sys.version_info >= (3, 7):
 if sys.version_info > (3, 7):
     standard_library.remove('macpath')
 
+# 'dummy_threading' module has been removed from Python 3.9
+if sys.version_info > (3, 8):
+    standard_library.remove('dummy_threading')
+
+# 'symbol' and 'formatter' modules have been removed from Python 3.10
+if sys.version_info >= (3, 10):
+    standard_library.remove('symbol')
+    standard_library.remove('formatter')
+
+# 'binhex' module has been removed from Python 3.11
+if sys.version_info >= (3, 11):
+    standard_library.remove('binhex')
+
+# 'smtpd', 'asyncore' and 'asynchat' modules have been removed from Python 3.12
+# https://docs.python.org/dev/whatsnew/3.12.html
+if sys.version_info >= (3, 12):
+    standard_library.remove('distutils')
+    standard_library.remove('imp')
+    standard_library.remove('smtpd')
+    standard_library.remove('asyncore')
+    standard_library.remove('asynchat')
+
+# 'aifc', 'cgi', 'cgitb', 'chunk', 'crypt', 'imghdr', 'lib2to3', 'mailcap', 'nntplib',
+# 'pipes', 'sndhdr', 'sunau', 'telnetlib', 'uu' and 'xdrlib' modules have been removed
+# from Python 3.13
+# https://docs.python.org/dev/whatsnew/3.13.html
+if sys.version_info >= (3, 13):
+    standard_library.remove('aifc')
+    standard_library.remove('cgi')
+    standard_library.remove('cgitb')
+    standard_library.remove('chunk')
+    standard_library.remove('crypt')
+    standard_library.remove('imghdr')
+    standard_library.remove('lib2to3')
+    standard_library.remove('mailcap')
+    standard_library.remove('nntplib')
+    standard_library.remove('pipes')
+    standard_library.remove('sndhdr')
+    standard_library.remove('sunau')
+    standard_library.remove('telnetlib')
+    standard_library.remove('uu')
+    standard_library.remove('xdrlib')
+
 # Remove tkinter and Easter eggs
 excluded_modules = [
     'antigravity',
@@ -261,7 +305,6 @@ excluded_modules = [
 def check_missing_modules(expected_modules):
     missing = []
     for module in expected_modules:
-        print('Try to import module ', module)
         try:
             importlib.import_module(module)
         except:

tests/sources/python-sqlite3.py

@@ -0,0 +1,19 @@
+import sqlite3
+from sqlite3 import Error
+
+def create_connection(db_file):
+    """ create a database connection to a SQLite database """
+    conn = None
+    try:
+        print('Sqlite3 version: ', sqlite3.version)
+        conn = sqlite3.connect(db_file)
+        conn.enable_load_extension(True)
+    except Error as e:
+        print(e)
+        exit(1)
+    finally:
+        if conn:
+            conn.close()
+
+if __name__ == '__main__':
+    create_connection(r"pythonsqlite.db")

tests/sources/python-urlopen-https.py

@@ -0,0 +1,10 @@
+import sys
+
+if sys.version_info[0] == 2:
+    from urllib2 import urlopen
+else:
+    from urllib.request import urlopen
+
+response = urlopen("https://raw.githubusercontent.com/actions/python-versions/c641695f6a07526c18f10e374e503e649fef9427/.gitignore")
+data = response.read()
+assert len(data) == 140, len(data)

tests/sources/tcltk.py

@@ -0,0 +1,10 @@
+import tkinter
+import _tkinter
+
+header = _tkinter.TK_VERSION
+lib = tkinter.Tk().getvar('tk_version')
+
+if lib != header:
+  print('header version=' + header)
+  print('lib version=' + lib)
+  exit(1)