Update versions-manifest (#243)

Co-authored-by: Service account <no-reply@microsoft.com>

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @actions/setup-actions-team

.github/workflows/build-python-packages.yml

@@ -0,0 +1,220 @@
+name: Build Python package
+run-name: Generate Python ${{ inputs.VERSION || '3.11.0' }}
+on:
+  workflow_dispatch:
+    inputs:
+      VERSION:
+        description: 'Python version to build and upload'
+        default: '3.11.0'
+        required: true
+      PUBLISH_RELEASES:
+        description: 'Whether to publish releases'
+        required: true
+        type: boolean
+        default: false
+      PLATFORMS:
+        description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
+        required: true
+        default: 'ubuntu-20.04,ubuntu-22.04,macos-11_x64,macos-11_arm64,windows-2019_x64,windows-2019_x86'
+  pull_request:
+    paths-ignore:
+    - 'versions-manifest.json'
+    - 'LICENSE'
+    - '**.md'
+    branches:
+    - 'main'
+
+env:
+  VERSION: ${{ inputs.VERSION || '3.11.0' }}
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  generate_matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generate-matrix.outputs.matrix }}
+    steps:
+      - name: Generate execution matrix
+        id: generate-matrix
+        run: |
+          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,macos-11,macos-11_arm64,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
+          $matrix = @()
+
+          foreach ($configuration in $configurations) {
+            $parts = $configuration.Split("_")
+            $os = $parts[0]
+            $arch = if ($parts[1]) {$parts[1]} else {"x64"}
+            switch -wildcard ($os) {
+              "*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
+              "*macos*" { $platform = 'darwin' }
+              "*windows*" { $platform = 'win32' }
+            }
+            $matrix += @{
+              'platform' = $platform
+              'os' = $os
+              'arch' = $arch
+            }
+          }
+          echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT
+
+  build_python:
+    needs: generate_matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
+    runs-on: ${{ matrix.os }}
+    env:
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
+    steps:
+
+      - name: Check out repository code
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Build Python ${{ env.VERSION }}
+        run: |
+          ./builders/build-python.ps1 -Version $env:VERSION `
+                  -Platform ${{ matrix.platform }} -Architecture ${{ matrix.arch }}
+
+      - name: Publish artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ${{ runner.temp }}/artifact
+
+  test_python:
+    needs: [generate_matrix, build_python]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
+    runs-on: ${{ matrix.os }}
+    env:
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
+    steps:
+
+      - name: Check out repository code
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Fully cleanup the toolcache directory before testing
+        run: ./helpers/clean-toolcache.ps1 -ToolName "Python"
+
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ runner.temp }}
+
+      - name: Extract files
+        run: |
+          if ('${{ matrix.platform }}' -eq 'win32') {
+            $artifactName = "${{ env.ARTIFACT_NAME }}.zip"
+            7z.exe x "$artifactName" -y | Out-Null
+          } else {
+            $artifactName = "${{ env.ARTIFACT_NAME }}.tar.gz"
+            tar -xzf $artifactName
+          }
+        working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
+
+      - name: Apply build artifact to the local machine
+        run: |
+          if ('${{ matrix.platform }}' -eq 'win32') {
+            powershell ./setup.ps1
+            } else {
+              sh ./setup.sh
+              cp ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}/build_output.txt ${{ runner.temp }}
+            }
+        working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
+
+      - name: Setup Python ${{ env.VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.VERSION }}
+          architecture: ${{ matrix.arch }}
+
+      - name: Verbose sysconfig dump
+        if: runner.os == 'Linux' || runner.os == 'macOS'
+        run: python ./sources/python-config-output.py
+        working-directory: ${{ github.workspace }}/tests
+
+      - name: Verbose python binary links
+        if: runner.os == 'Linux' || runner.os == 'macOS'
+        run: |
+          $pythonLocation = which python
+          if ('${{ matrix.platform }}' -eq 'darwin') { otool -L $pythonLocation } else { ldd $pythonLocation }
+
+      - name: Run tests
+        run: |
+            Install-Module Pester -Force -Scope CurrentUser
+            Import-Module Pester
+            $pesterContainer = New-PesterContainer -Path './python-tests.ps1' -Data @{
+                Version="${{ env.VERSION }}";
+                Platform="${{ matrix.platform }}";
+                Architecture="${{ matrix.arch }}";
+            }
+            $Result = Invoke-Pester -Container $pesterContainer -PassThru
+            if ($Result.FailedCount -gt 0) {
+              $host.SetShouldExit($Result.FailedCount)
+              exit $Result.FailedCount
+            }
+        working-directory: ${{ github.workspace }}/tests
+
+  publish_release:
+      name: Publish release
+      if: github.event_name == 'workflow_dispatch' && inputs.PUBLISH_RELEASES
+      needs: test_python
+      runs-on: ubuntu-latest
+      steps:
+      - uses: actions/download-artifact@v3
+
+      - name: Publish Release ${{ env.VERSION }}
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.VERSION }}-${{ github.run_id }}
+          release_name: ${{ env.VERSION }}
+          body: |
+            Python ${{ env.VERSION }}
+
+      - name: Upload release assets
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            for (let artifactDir of fs.readdirSync('.')) {
+              let artifactName = fs.readdirSync(`${artifactDir}`)[0];
+              console.log(`Upload ${artifactName} asset`);
+              github.rest.repos.uploadReleaseAsset({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                release_id: ${{ steps.create_release.outputs.id }},
+                name: artifactName,
+                data: fs.readFileSync(`./${artifactDir}/${artifactName}`)
+              });
+            }
+
+  trigger_pr:
+    name: Trigger "Create Pull Request" workflow
+    needs: publish_release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Trigger "Create Pull Request" workflow
+      uses: actions/github-script@v6
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          github.rest.actions.createWorkflowDispatch({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            workflow_id: 'create-pr.yml',
+            ref: 'main'
+          });
+

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,16 @@
+name: CodeQL analysis
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 3 * * 0'
+
+jobs:
+  call-codeQL-analysis:
+    name: CodeQL analysis 
+    uses: actions/reusable-workflows/.github/workflows/codeql-analysis.yml@main
+    with:
+     languages: "['python']"

.github/workflows/create-pr.yml

@@ -0,0 +1,10 @@
+name: Create Pull Request
+on:
+  workflow_dispatch:
+
+jobs:
+  create-pr:
+    uses: actions/versions-package-tools/.github/workflows/create-pr-to-update-manifest.yml@main
+    with:
+      tool-name: "python"
+    secrets: inherit

.github/workflows/create-release.yml

@@ -0,0 +1,18 @@
+name: Create release
+on:
+  repository_dispatch:
+    types: [create-release]
+
+jobs:
+  create_release:
+    name: Create release ${{ github.event.client_payload.ToolVersion }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Create release for Python ${{ github.event.client_payload.ToolVersion }}
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ github.event.client_payload.TagName }}
+        release_name: ${{ github.event.client_payload.ToolVersion }}
+        body: ${{ github.event.client_payload.ReleaseBody }}

.github/workflows/get-python-versions.yml

@@ -0,0 +1,13 @@
+name: Get Python versions
+on:
+  schedule:
+    - cron: '0 3,15 * * *'
+  workflow_dispatch:
+
+jobs:
+  get-new-python-versions:
+    uses: actions/versions-package-tools/.github/workflows/get-new-tool-versions.yml@main
+    with:
+      tool-name: "Python"
+      image-url: "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+    secrets: inherit

.github/workflows/manifest-config-validation.yml

@@ -0,0 +1,32 @@
+name: Manifest config tests
+on: 
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - 'versions-manifest.json'
+
+jobs:
+  RunTests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        submodules: true
+
+    - name: Install Pester
+      shell: pwsh
+      run: |
+        Install-Module Pester -Force -Scope CurrentUser
+
+    - name: Run tests
+      shell: pwsh
+      run: |
+        Import-Module Pester
+        Invoke-Pester -Configuration @{
+          Run = @{
+            Path = "tests/ManifestConfig.Tests.ps1"
+            Exit = $true
+          }
+        }

.github/workflows/python-versions-runner.yml

@@ -0,0 +1,31 @@
+name: Python versions runner
+on:
+  workflow_dispatch:
+    inputs:
+      versions:
+        description: 'Versions to build'
+        required: true
+        default: '","'
+      publish-releases:
+        description: 'Whether to publish releases'
+        required: true
+        default: 'false'
+
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  trigger_builds:
+    name: Trigger python build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      
+      - name: Trigger python workflow
+        run: |
+          $versions = ${{ github.event.inputs.versions }}
+          ./builders/python-versions-runner.ps1 -Versions $versions.Split(",") -PublishRelease ${{ github.event.inputs.publish-releases }}
+        env:
+          TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/releases-validation.yml

@@ -0,0 +1,36 @@
+name: Validate 'versions-manifest.json' file
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'versions-manifest.json'
+
+jobs:
+  setup-versions-from-manifest:
+    name: Setup ${{ matrix.python }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04]
+        python: [3.9.12, 3.10.8, 3.11.10]
+    steps:
+    - name: setup-python ${{ matrix.python }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Validate version
+      run: |
+        $pythonVersion = (python --version)
+        if ("Python ${{ matrix.python }}" -ne "$pythonVersion"){
+          Write-Host "The current version is $pythonVersion; expected version is ${{ matrix.python }}"
+          exit 1
+        }
+        $pythonVersion
+      shell: pwsh
+
+    - name: Run simple code
+      run: python -c 'import math; print(math.factorial(5))'

.github/workflows/test-python-version.yml

@@ -0,0 +1,34 @@
+name: Test Python version
+on: 
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Python version to build'
+        required: true
+      architecture:
+        description: 'The target architecture (x86, x64) of the Python'
+        required: false
+        default: 'x64'
+
+jobs:
+  test-python:
+    name: Test Python ${{ github.event.inputs.version }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04] 
+    steps:
+    - name: Setup Python ${{ github.event.inputs.version }}
+      uses: actions/setup-python@main
+      with:
+        python-version: ${{ github.event.inputs.version }}
+        architecture: ${{ github.event.inputs.architecture }}
+
+    - name: Validate version
+      run: |
+        python --version
+      shell: pwsh
+
+    - name: Run simple code
+      run: python -c 'import math; print(math.factorial(5))'

.github/workflows/validate-manifest.yml

@@ -0,0 +1,21 @@
+name: Validate manifest
+on:
+# The GITHUB_TOKEN secret is used to create a PR
+# The pull_request event will not be triggered by it
+# That's one of the reasons we need the schedule to validate the versions-manifest.json file
+  schedule:
+    - cron: '0 8,20 * * *'
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'versions-manifest.json'
+
+jobs:
+  manifest:
+    uses: actions/versions-package-tools/.github/workflows/validate-manifest.yml@main
+    with:
+      tool-name: "Python"
+      image-url: "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+    secrets: inherit

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "helpers"]
+	path = helpers
+	url = https://github.com/actions/versions-package-tools
+	branch = main

README.md

@@ -1,16 +1,25 @@
 # Python for Actions
-This repository contains the code and scripts that we use to build Python packages used in [virtual-environments](https://github.com/actions/virtual-environments) and accessible through the [setup-python](https://github.com/actions/setup-python) Action.  
-File [versions-manifest.json](./versions-manifest.json) contains the list of available and released versions.  
+This repository contains the code and scripts that we use to build Python packages used in [runner-images](https://github.com/actions/runner-images) and accessible through the [setup-python](https://github.com/actions/setup-python) Action.
+File [versions-manifest.json](./versions-manifest.json) contains the list of available and released versions.
 
-> Caution: this is prepared for and only permitted for use by actions `virtual-environments` and `setup-python` action.
+> Caution: this is prepared for and only permitted for use by actions `runner-images` and `setup-python` action.
 
-**Status**: Currently under development and in use for beta and preview actions.  This repo is undergoing rapid changes.
+Some versions are pre-installed on [runner-images](https://github.com/actions/runner-images) images.
+More versions are available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
 
-Some versions are pre-installed on [virtual-environments](https://github.com/actions/virtual-environments) images.  
-More versions will (soon!) be available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
+## Building installation packages
+**Ubuntu:** The official Python sources are downloaded from [python.org](https://www.python.org/ftp/python/), built using the `make` tool, and archived along with the installation script for further distribution and installation. We build each Python version against all [versions of Ubuntu](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) that are available at the time the Python version is released.
 
-## Adding new versions
-We are trying to build and release new versions of Python as soon as they are released. Please open an issue if any versions are missing.
+**macOS:** 
+- For **Python < 3.11**, the official Python sources are downloaded from [python.org](https://www.python.org/ftp/python/), built using the `make` tool, and archived along with the installation script for further distribution and installation.  For backward compatibility, we build each Python version against the oldest [version of macOS](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) that is available at the time the Python version is released.
+- For **Python >= 3.11**, the official macOS `universal2` Python binaries are simply downloaded from [python.org](https://www.python.org/ftp/python/) and archived along with the installation script for further distribution and installation.
+
+**Windows:** The official Python executables are simply downloaded from [python.org](https://www.python.org/ftp/python/) and archived along with the installation script for further distribution and installation.
+
+## Support policy
+We aim to make new versions of Python available as soon as they are released. Both stable and prerelease versions are considered for building and releasing. Please open an issue in [actions/setup-python](https://github.com/actions/setup-python) if a version you're looking for isn't available.
+
+When a new version of an operating system is released and made available for use with [GitHub hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources), we will provide the **latest existing patch versions of Python for all major versions that have not reached [end-of-life](https://devguide.python.org/versions/)**.  Any subsequent Python versions will be made available for the new OS as well.
 
 ## Contribution
 Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure

azure-pipelines/build-python-packages.yml

@@ -1,99 +0,0 @@
-name: $(date:yyyyMMdd)$(rev:.r)-Python-$(VERSION)
-trigger: none
-pr: none
-
-stages:
-- stage: Build_Python_MacOS
-  dependsOn: []
-  variables:
-    VmImage: 'macOS-10.14'
-    Platform: macos-1014
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_MacOS
-  condition: succeeded()
-  dependsOn: Build_Python_MacOS
-  variables:
-    VmImage: 'macOS-10.14'
-    Platform: macos-1014
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_Ubuntu_1604
-  dependsOn: []
-  variables:
-    VmImage: 'ubuntu-16.04'
-    Platform: ubuntu-1604
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_Ubuntu_1604
-  condition: succeeded()
-  dependsOn: Build_Python_Ubuntu_1604
-  variables:
-    VmImage: 'ubuntu-16.04'
-    Platform: ubuntu-1604
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_Ubuntu_1804
-  dependsOn: []
-  variables:
-    VmImage: 'ubuntu-18.04'
-    Platform: ubuntu-1804
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_Ubuntu_1804
-  condition: succeeded()
-  dependsOn: Build_Python_Ubuntu_1804
-  variables:
-    VmImage: 'ubuntu-18.04'
-    Platform: ubuntu-1804
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_X64_Windows
-  dependsOn: []
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: windows-2016
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_x64_Windows
-  condition: succeeded()
-  dependsOn: Build_Python_X64_Windows
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: windows-2016
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_x86_Windows
-  dependsOn: []
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: windows-2016
-    Architecture: x86
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_x86_Windows
-  condition: succeeded()
-  dependsOn: Build_Python_x86_Windows
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: windows-2016
-    Architecture: x86
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml

azure-pipelines/run-ci-builds.yml

@@ -1,34 +0,0 @@
-trigger: none
-pr:
-  autoCancel: true
-  branches:
-    include:
-    - master
-  paths:
-    exclude:
-    - versions-manifest.json
-
-jobs:
-- job: Run_Builds
-  pool:
-    name: Azure Pipelines
-    vmImage: 'ubuntu-latest'
-  timeoutInMinutes: 180
-
-  steps:
-  - checkout: self
-    submodules: true
-
-  - task: PowerShell@2
-    displayName: 'Run build'
-    inputs:
-      targetType: filePath
-      filePath: './helpers/azure-devops/run-ci-builds.ps1 '
-      arguments: |
-          -TeamFoundationCollectionUri $(System.TeamFoundationCollectionUri) `
-          -AzureDevOpsProjectName $(System.TeamProject) `
-          -AzureDevOpsAccessToken $(System.AccessToken) `
-          -SourceBranch $(Build.SourceBranch) `
-          -SourceVersion $(Build.SourceVersion) `
-          -ToolVersions "$(PYTHON_VERSIONS)" `
-          -DefinitionId $(DEFINITION_ID) 

azure-pipelines/templates/build-job.yml

@@ -1,22 +0,0 @@
-jobs:
-- job: Build_Python
-  timeoutInMinutes: 90
-  pool:
-    name: Azure Pipelines
-    vmImage: $(VmImage)
-  steps:
-  - checkout: self
-    submodules: true
-    
-  - task: PowerShell@2
-    displayName: 'Build Python $(VERSION)'
-    inputs:
-      targetType: filePath
-      filePath: './builders/build-python.ps1'
-      arguments: '-Version $(VERSION) -Platform $(Platform) -Architecture $(Architecture)'
-
-  - task: PublishPipelineArtifact@1
-    displayName: 'Publish Artifact: Python $(VERSION)'
-    inputs:
-      targetPath: '$(Build.ArtifactStagingDirectory)' 
-      artifactName: 'python-$(VERSION)-$(Platform)-$(Architecture)'

azure-pipelines/templates/test-job.yml

@@ -1,80 +0,0 @@
-jobs:
-- job: Test_Python
-  pool: 
-    name: Azure Pipelines
-    vmImage: $(VmImage)
-  steps:
-  - checkout: self
-    submodules: true
-
-  - task: DownloadPipelineArtifact@2
-    inputs:
-      source: 'current'
-      artifact: 'python-$(VERSION)-$(Platform)-$(Architecture)'
-      path: $(Build.BinariesDirectory)
-
-  - task: ExtractFiles@1
-    inputs:
-      archiveFilePatterns: '$(Build.BinariesDirectory)/python-$(VERSION)-$(Platform)-$(Architecture).*'
-      destinationFolder: $(Build.BinariesDirectory)
-      cleanDestinationFolder: false
-
-  - task: PowerShell@2
-    displayName: 'Apply build artifact to the local machines'
-    inputs:
-      TargetType: inline
-      script: |
-        if ($env:PLATFORM -match 'windows') { powershell ./setup.ps1 } else { sh ./setup.sh }
-      workingDirectory: '$(Build.BinariesDirectory)'
-
-  - task: UsePythonVersion@0
-    displayName: 'Use Python $(VERSION)'
-    inputs:
-      versionSpec: '$(VERSION)'
-      architecture: '$(Architecture)'
-
-  - task: PowerShell@2
-    displayName: 'Verbose sysconfig dump'
-    inputs:
-      TargetType: inline
-      script: |
-        Invoke-Expression "python ./sources/python-config-output.py"
-      workingDirectory: '$(Build.SourcesDirectory)/tests'
-    condition: ne(variables['Platform'], 'windows-2016')
-
-  - task: PowerShell@2
-    displayName: 'Verbose python binary links'
-    inputs:
-      TargetType: inline
-      script: |
-        $pythonLocation = which python
-        if ($env:PLATFORM -match 'macos') { otool -L $pythonLocation } else { ldd $pythonLocation }
-      workingDirectory: '$(Build.BinariesDirectory)'
-    condition: ne(variables['Platform'], 'windows-2016')
-
-  - task: PowerShell@2
-    displayName: 'Run tests'
-    inputs:
-      TargetType: inline
-      script: |
-        Install-Module Pester -Force -Scope CurrentUser
-        Import-Module Pester
-        $pesterParams = @{
-          Path="./python-tests.ps1";
-          Parameters=@{
-            Version="$(VERSION)";
-            Platform="$(Platform)";
-          }
-        }
-        Invoke-Pester -Script $pesterParams -OutputFile "test_results.xml" -OutputFormat NUnitXml
-      workingDirectory: '$(Build.SourcesDirectory)/tests'
-
-  - task: PublishTestResults@2
-    displayName: 'Publish test results'
-    inputs:
-      testResultsFiles: '*.xml'
-      testResultsFormat: NUnit
-      searchFolder: 'tests'
-      failTaskOnFailedTests: true
-      testRunTitle: "$(TestRunTitle)"
-    condition: always()

builders/build-python.ps1

@@ -1,6 +1,6 @@
-using module "./builders/win-python-builder.psm1"
-using module "./builders/ubuntu-python-builder.psm1"
-using module "./builders/macos-python-builder.psm1"
+using module "./win-python-builder.psm1"
+using module "./ubuntu-python-builder.psm1"
+using module "./macos-python-builder.psm1"
 
 <#
 .SYNOPSIS
@@ -21,7 +21,7 @@ Required parameter. The platform for which Python will be built.
 #>
 
 param(
-    [Parameter (Mandatory=$true)][Version] $Version,
+    [Parameter (Mandatory=$true)][semver] $Version,
     [Parameter (Mandatory=$true)][string] $Platform,
     [string] $Architecture = "x64"
 )
@@ -29,6 +29,7 @@ param(
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "common-helpers.psm1") -DisableNameChecking
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "nix-helpers.psm1") -DisableNameChecking
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "win-helpers.psm1") -DisableNameChecking
+Import-Module (Join-Path $PSScriptRoot "python-version.psm1") -DisableNameChecking
 
 function Get-PythonBuilder {
     <#
@@ -49,18 +50,17 @@ function Get-PythonBuilder {
 
     #>
 
-    param (
-        [version] $Version,
+    param(
+        [semver] $Version,
         [string] $Architecture,
         [string] $Platform
     )
 
-    $Platform = $Platform.ToLower()  
-    if ($Platform -match 'windows') {
+    if ($Platform -match 'win32') {
         $builder = [WinPythonBuilder]::New($Version, $Architecture, $Platform)
-    } elseif ($Platform -match 'ubuntu') {
+    } elseif ($Platform -match 'linux') {
         $builder = [UbuntuPythonBuilder]::New($Version, $Architecture, $Platform)
-    } elseif ($Platform -match 'macos') {
+    } elseif ($Platform -match 'darwin') {
         $builder = [macOSPythonBuilder]::New($Version, $Architecture, $Platform)
     } else {
         Write-Host "##vso[task.logissue type=error;] Invalid platform: $Platform"
@@ -71,5 +71,5 @@ function Get-PythonBuilder {
 }
 
 ### Create Python builder instance, and build artifact
-$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture  -Platform $Platform 
+$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture -Platform $Platform 
 $Builder.Build()

builders/invoke-workflow.psm1

@@ -0,0 +1,52 @@
+function Invoke-Workflow {
+    param (
+        [string] $Version,
+        [string] $PublishRelease
+    )
+    
+    $payload = @{
+        "ref" = "main"
+        "inputs" = @{
+            "VERSION" = "$Version"
+            "PUBLISH_RELEASES" = "$PublishRelease"
+        }
+    } | ConvertTo-Json
+    $headers = @{
+        Authorization="Bearer $env:TOKEN"
+    }
+    $actionsRepoUri = "$env:GITHUB_API_URL/repos/$env:GITHUB_REPOSITORY/actions"
+    Invoke-RestMethod -uri "$actionsRepoUri/workflows/build-python-packages.yml/dispatches" -method POST -headers $headers -body $payload
+
+    $result = [PSCustomObject]@{
+        Version = $Version
+        Conclusion = "failure"
+        Url = "Not run"
+    }
+    # Triggering workflow and verifying that it has been triggered with retries
+    while (-not $workflowToCheck) {
+        Start-Sleep -seconds 40
+        $workflowRuns = (Invoke-RestMethod "$actionsRepoUri/runs").workflow_runs | Where-Object {$_.status -like "*progress*" -and $_.id -ne $env:GITHUB_RUN_ID}
+        $workflowToCheck = $workflowRuns | Where-Object {
+            (Invoke-RestMethod "$actionsRepoUri/runs/$($_.id)/jobs").jobs.steps.name -like "*$Version"
+        }
+        $retries++
+        if ($retries -gt 10) {
+            Write-Host "Workflow triggered for version '$Version' not found or something went wrong with fetching the workflow status"
+            return $result
+        }
+    }
+    # Waiting for workflow to complete
+    while ($workflowToCheck.status -ne "completed") {
+        Start-Sleep -Seconds 120
+        $workflowToCheck = Invoke-RestMethod "$actionsRepoUri/runs/$($workflowToCheck.id)"
+        Write-Host "Workflow run with Id: $($workflowToCheck.id) for version '$Version' - status '$($workflowToCheck.status)'"
+    }
+    $result.Conclusion = $workflowToCheck.conclusion
+    $result.Url = $workflowToCheck.html_url
+    if ($workflowToCheck.conclusion -ne "success") {
+        Write-Host "Triggered workflow for version '$Version' completed unsuccessfully with result '$($workflowToCheck.conclusion)'. Check the logs: $($workflowToCheck.html_url)"
+        return $result
+    }
+    Write-Host "Triggered workflow for version '$Version' succeeded; Url: $($workflowToCheck.html_url)"
+    return $result
+}

builders/macos-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/nix-python-builder.psm1"
+using module "./nix-python-builder.psm1"
 
 class macOSPythonBuilder : NixPythonBuilder {
     <#
@@ -8,6 +8,10 @@ class macOSPythonBuilder : NixPythonBuilder {
     .DESCRIPTION
     Contains methods that required to build macOS Python artifact from sources. Inherited from base NixPythonBuilder.
 
+    While python.org provides precompiled binaries for macOS, switching to them risks breaking existing customers.
+    If we wanted to start using the official binaries instead of building from source, we should avoid changing previous versions
+    so we remain backwards compatible.
+
     .PARAMETER platform
     The full name of platform for which Python should be built.
 
@@ -17,11 +21,26 @@ class macOSPythonBuilder : NixPythonBuilder {
     #>
 
     macOSPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) { }
 
+    [void] PrepareEnvironment() {
+        <#
+        .SYNOPSIS
+        Prepare system environment by installing dependencies and required packages.
+        #>
+        
+        if ($this.Version -eq "3.7.17") {
+            # We have preinstalled ncurses and readLine on the hoster runners. But we need to install bzip2 for 
+            # setting up an environemnt 
+            # If we get any issues realted to ncurses or readline we can try to run this command
+            # brew install ncurses readline
+            Execute-Command -Command "brew install bzip2"
+        }
+    }
+
     [void] Configure() {
         <#
         .SYNOPSIS
@@ -29,30 +48,149 @@ class macOSPythonBuilder : NixPythonBuilder {
         #>
 
         $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation()
-        $configureString = "./configure --prefix=$pythonBinariesLocation --enable-optimizations --enable-shared --with-lto"
+        $configureString = "./configure"
+        $configureString += " --prefix=$pythonBinariesLocation"
+        $configureString += " --enable-optimizations"
+        $configureString += " --enable-shared"
+        $configureString += " --with-lto"
+
+        ### For Python versions which support it, compile a universal2 (arm64 + x86_64 hybrid) build. The arm64 slice
+        ### will never be used itself by a Github Actions runner but using a universal2 Python is the only way to build
+        ### universal2 C extensions and wheels. This is supported by Python >= 3.10 and was backported to Python >=
+        ### 3.9.1 and >= 3.8.10.
+        ### Disabled, discussion: https://github.com/actions/python-versions/pull/114
+        # if ($this.Version -ge "3.8.10" -and $this.Version -ne "3.8.13" -and $this.Version -ne "3.9.0" ) {
+        #     $configureString += " --enable-universalsdk --with-universal-archs=universal2"
+        # }
 
         ### OS X 10.11, Apple no longer provides header files for the deprecated system version of OpenSSL.
         ### Solution is to install these libraries from a third-party package manager,
         ### and then add the appropriate paths for the header and library files to configure command.
         ### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies)
         if ($this.Version -lt "3.7.0") {
-            $env:LDFLAGS="-L$(brew --prefix openssl)/lib"
-            $env:CFLAGS="-I$(brew --prefix openssl)/include"
+            $env:LDFLAGS = "-L/usr/local/opt/openssl@1.1/lib -L/usr/local/opt/zlib/lib"
+            $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/zlib/include"
         } else {
-            $configureString += " --with-openssl=/usr/local/opt/openssl"
+            $configureString += " --with-openssl=/usr/local/opt/openssl@1.1"
+
+            # For Python 3.7.2 and 3.7.3 we need to provide PATH for zlib to pack it properly. Otherwise the build will fail
+            # with the error: zipimport.ZipImportError: can't decompress data; zlib not available
+            if ($this.Version -eq "3.7.2" -or $this.Version -eq "3.7.3" -or $this.Version -eq "3.7.17") {
+                $env:LDFLAGS = "-L/usr/local/opt/zlib/lib"
+                $env:CFLAGS = "-I/usr/local/opt/zlib/include"
+            }
+
+            if ($this.Version -gt "3.7.12") {
+                $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'"
+	        }
+
+            if ($this.Version -eq "3.7.17") {
+                $env:LDFLAGS += " -L$(brew --prefix bzip2)/lib -L$(brew --prefix readline)/lib -L$(brew --prefix ncurses)/lib"
+                $env:CFLAGS += " -I$(brew --prefix bzip2)/include -I$(brew --prefix readline)/include -I$(brew --prefix ncurses)/include"
+            }
         }
 
+        ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
+        ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
+        if ($this.Version -ge "3.2.0") {
+            $configureString += " --enable-loadable-sqlite-extensions"
+            $env:LDFLAGS += " -L$(brew --prefix sqlite3)/lib"
+            $env:CFLAGS += " -I$(brew --prefix sqlite3)/include"
+            $env:CPPFLAGS += "-I$(brew --prefix sqlite3)/include"
+        }
+
+        Write-Host "The passed configure options are: "
+        Write-Host $configureString
+
         Execute-Command -Command $configureString
     }
 
-    [void] PrepareEnvironment() {
+    [string] GetPkgName() {
         <#
         .SYNOPSIS
-        Prepare system environment by installing dependencies and required packages.
+        Return Python installation Package.
         #>
 
-        ### reinstall header files to Avoid issue with X11 headers on Mojave
-        $pkgName = "/Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg"
-        Execute-Command -Command "sudo installer -pkg $pkgName -target /"
+        $nativeVersion = Convert-Version -version $this.Version
+        $architecture = "-macos11"
+        $extension = ".pkg"
+
+        $pkg = "python-${nativeVersion}${architecture}${extension}"
+
+        return $pkg
+    }
+
+    [uri] GetPkgUri() {
+        <#
+        .SYNOPSIS
+        Get base Python URI and return complete URI for Python installation package.
+        #>
+
+        $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $pkg = $this.GetPkgName()
+
+        $uri = "${base}/${versionName}/${pkg}"
+
+        return $uri
+    }
+
+    [string] DownloadPkg() {
+        <#
+        .SYNOPSIS
+        Download Python installation executable into artifact location.
+        #>
+
+        $pkgUri = $this.GetPkgUri()
+
+        Write-Host "Sources URI: $pkgUri"
+        $pkgLocation = Download-File -Uri $pkgUri -OutputFolder $this.WorkFolderLocation
+        Write-Debug "Done; Package location: $pkgLocation"
+
+        New-Item -Path $this.WorkFolderLocation -Name "build_output.txt"  -ItemType File
+        return $pkgLocation
+    }
+
+    [void] CreateInstallationScriptPkg() {
+        <#
+        .SYNOPSIS
+        Create Python artifact installation script based on specified template.
+        #>
+
+        $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath "macos-pkg-setup-template.sh"
+        $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
+        $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName  -ItemType File
+
+        $variablesToReplace = @{
+            "{{__VERSION_FULL__}}" = $this.Version;
+            "{{__PKG_NAME__}}" = $this.GetPkgName();
+            "{{__ARCH__}}" = $this.Architecture;
+        }
+
+        $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
+        $installationTemplateContent | Out-File -FilePath $installationScriptLocation
+        Write-Debug "Done; Installation script location: $installationScriptLocation)"
+    }
+
+    [void] Build() {
+        <#
+        .SYNOPSIS
+        Generates Python artifact from downloaded Python installation executable.
+        #>
+
+        $PkgVersion = [semver]"3.11.0-beta.1"
+
+        if (($this.Version -ge $PkgVersion) -or ($this.Architecture -eq "arm64")) {
+            Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..."
+            $this.DownloadPkg()
+
+            Write-Host "Create installation script..."
+            $this.CreateInstallationScriptPkg()
+        } else {
+            ([NixPythonBuilder]$this).Build()
+        }
+
+        Write-Host "Archive artifact"
+        $this.ArchiveArtifact()
     }
 }

builders/nix-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/python-builder.psm1"
+using module "./python-builder.psm1"
 
 class NixPythonBuilder : PythonBuilder {
     <#
@@ -33,13 +33,12 @@ class NixPythonBuilder : PythonBuilder {
     [string] $OutputArtifactName
 
     NixPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) {
         $this.InstallationTemplateName = "nix-setup-template.sh"	
         $this.InstallationScriptName = "setup.sh"
-
         $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.tar.gz"
     }
 
@@ -50,8 +49,10 @@ class NixPythonBuilder : PythonBuilder {
         #>
 
         $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $nativeVersion = Convert-Version -version $this.Version
 
-        return "${base}/$($this.Version)/Python-$($this.Version).tgz"
+        return "${base}/${versionName}/Python-${nativeVersion}.tgz"
     }
 
     [string] GetPythonBinary() {
@@ -95,9 +96,7 @@ class NixPythonBuilder : PythonBuilder {
         $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
 
         $variablesToReplace = @{
-            "{{__VERSION_MAJOR__}}" = $this.Version.Major;
-            "{{__VERSION_MINOR__}}" = $this.Version.Minor;
-            "{{__VERSION_BUILD__}}" = $this.Version.Build;
+            "{{__VERSION_FULL__}}" = $this.Version;
         }
         $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
 

builders/python-builder.psm1

@@ -29,7 +29,7 @@ class PythonBuilder {
 
     #>
 
-    [version] $Version
+    [semver] $Version
     [string] $Architecture
     [string] $Platform
     [string] $HostedToolcacheLocation
@@ -38,17 +38,23 @@ class PythonBuilder {
     [string] $ArtifactFolderLocation
     [string] $InstallationTemplatesLocation
 
-    PythonBuilder ([version] $version, [string] $architecture, [string] $platform) {
+    PythonBuilder ([semver] $version, [string] $architecture, [string] $platform) {
+        $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers"
+
+        $artifactDirectory = Join-Path $env:RUNNER_TEMP "artifact"
+        $workDirectory = Join-Path $env:RUNNER_TEMP "work"
+
+        New-Item -Force -Type Directory $artifactDirectory
+        New-Item -Force -Type Directory $workDirectory
+
+        $this.HostedToolcacheLocation = $env:RUNNER_TOOL_CACHE
+        $this.TempFolderLocation = $env:RUNNER_TEMP
+        $this.WorkFolderLocation =  $workDirectory
+        $this.ArtifactFolderLocation = $artifactDirectory
+
         $this.Version = $version
         $this.Architecture = $architecture
         $this.Platform = $platform
-
-        $this.HostedToolcacheLocation = $env:AGENT_TOOLSDIRECTORY
-        $this.TempFolderLocation = $env:BUILD_SOURCESDIRECTORY
-        $this.WorkFolderLocation = $env:BUILD_BINARIESDIRECTORY
-        $this.ArtifactFolderLocation = $env:BUILD_STAGINGDIRECTORY
-
-        $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers"
     }
 
     [uri] GetBaseUri() {
@@ -79,11 +85,21 @@ class PythonBuilder {
         return "$pythonToolcacheLocation/$($this.Version)/$($this.Architecture)"
     }
 
+    [string] GetBaseVersion() {
+        <#
+        .SYNOPSIS
+        Return Major.Minor.Patch version string.
+        #>
+
+        return "$($this.Version.Major).$($this.Version.Minor).$($this.Version.Patch)"
+    }
+
     [void] PreparePythonToolcacheLocation() {
         <#
         .SYNOPSIS
         Prepare system hostedtoolcache folder for new Python version. 
         #>
+        
         $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation()
 
         if (Test-Path $pythonBinariesLocation) {

builders/python-version.psm1

@@ -0,0 +1,45 @@
+function Convert-Label() {
+    <#
+    .SYNOPSIS
+    Convert generic semver label to native Python label.
+    #>
+
+    param(
+        [Parameter(Mandatory)]
+        [string] $label
+    )
+
+    switch ($label) {
+        "alpha" { return "a" }
+        "beta" { return "b" }
+        "rc" { return "rc" }
+        default { throw "Invalid version label '$label'" }
+    }
+}
+
+function Convert-Version {
+    <#
+    .SYNOPSIS
+    Convert generic semver version to native Python version.
+    #>
+
+    param(
+        [Parameter(Mandatory)]
+        [semver] $version,
+        [char] $delimiter = "."
+    )
+
+    $nativeVersion = "{0}.{1}.{2}" -f $version.Major, $version.Minor, $version.Patch
+
+    if ($version.PreReleaseLabel)
+    {
+        $preReleaseLabel = $version.PreReleaseLabel.Split($delimiter)
+        
+        $preReleaseLabelName = Convert-Label -Label $preReleaseLabel[0]
+        $preReleaseLabelVersion = $preReleaseLabel[1]
+
+        $nativeVersion += "${preReleaseLabelName}${preReleaseLabelVersion}"
+    }
+
+    return $nativeVersion
+}

builders/python-versions-runner.ps1

@@ -0,0 +1,31 @@
+<#
+.SYNOPSIS
+Generate Python artifact.
+
+.DESCRIPTION
+Script that triggering and fetching the result of the "Build python package" workflows with provided python versions
+
+.PARAMETER Version
+Required parameter. Python versions to trigger builds for.
+
+.PARAMETER PublishRelease
+Switch parameter. Whether to publish release for built version.
+
+#>
+
+param(
+    [Parameter (Mandatory=$true, HelpMessage="Python version to trigger build for")]
+    [array] $Versions,
+    [Parameter (Mandatory=$false, HelpMessage="Whether to publish release for built version")]
+    [string] $PublishRelease
+)
+
+$summary = $Versions | ForEach-Object -Parallel { 
+    Import-Module "./builders/invoke-workflow.psm1"
+    Invoke-Workflow -Version $_ -PublishRelease $Using:PublishRelease
+}
+Write-Host "Results of triggered workflows:"
+$summary | Out-String
+if ($summary.Conclusion -contains "failure" -or $summary.Conclusion -contains "cancelled") {
+    exit 1
+}

builders/ubuntu-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/nix-python-builder.psm1"
+using module "./nix-python-builder.psm1"
 
 class UbuntuPythonBuilder : NixPythonBuilder {
     <#
@@ -17,7 +17,7 @@ class UbuntuPythonBuilder : NixPythonBuilder {
     #>
 
     UbuntuPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) { }
@@ -32,13 +32,25 @@ class UbuntuPythonBuilder : NixPythonBuilder {
 
         ### To build Python with SO we must pass full path to lib folder to the linker
         $env:LDFLAGS="-Wl,--rpath=${pythonBinariesLocation}/lib"
-        $configureString = "./configure --prefix=$pythonBinariesLocation --enable-shared --enable-optimizations"
+        $configureString = "./configure"
+        $configureString += " --prefix=$pythonBinariesLocation"
+        $configureString += " --enable-shared"
+        $configureString += " --enable-optimizations"
 
+        ### Compile with ucs4 for Python 2.x. On 3.x, ucs4 is enabled by default
         if ($this.Version -lt "3.0.0") {
-            ### Compile with ucs4 for Python 2.x. On 3.x, ucs4 is enabled by default
             $configureString += " --enable-unicode=ucs4"
         }
 
+        ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
+        ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
+        if ($this.Version -ge "3.2.0") {
+            $configureString += " --enable-loadable-sqlite-extensions"
+        }
+
+        Write-Host "The passed configure options are: "
+        Write-Host $configureString
+
         Execute-Command -Command $configureString
     }
 
@@ -59,6 +71,8 @@ class UbuntuPythonBuilder : NixPythonBuilder {
         } else {
             $tkinterInstallString = "sudo apt install -y python-tk tk-dev"
         }
+
+        Execute-Command -Command "sudo apt-get update"
         Execute-Command -Command $tkinterInstallString
 
         ### Install dependent packages
@@ -76,10 +90,7 @@ class UbuntuPythonBuilder : NixPythonBuilder {
             Execute-Command -Command "sudo apt install -y $_"
         }
 
-        if ($this.Platform -ne "ubuntu-1604") {
-            ### On Ubuntu-1804, libgdbm-compat-dev has older modules that are no longer in libgdbm-dev
-            Execute-Command -Command "sudo apt install -y libgdbm-compat-dev"
-        }
-        
+        ### On Ubuntu-1804, libgdbm-compat-dev has older modules that are no longer in libgdbm-dev
+        Execute-Command -Command "sudo apt install -y libgdbm-compat-dev"
     }
 }

builders/win-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/python-builder.psm1"
+using module "./python-builder.psm1"
 
 class WinPythonBuilder : PythonBuilder {
     <#
@@ -27,7 +27,7 @@ class WinPythonBuilder : PythonBuilder {
     [string] $OutputArtifactName
 
     WinPythonBuilder(
-        [version] $version,
+        [semver] $version,
         [string] $architecture,
         [string] $platform
     ) : Base($version, $architecture, $platform) {
@@ -72,10 +72,12 @@ class WinPythonBuilder : PythonBuilder {
         #>
 
         $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $nativeVersion = Convert-Version -version $this.Version
         $architecture = $this.GetArchitectureExtension()
         $extension = $this.GetPythonExtension()
 
-        $uri = "${base}/$($this.Version)/python-$($this.Version)${architecture}${extension}"
+        $uri = "${base}/${versionName}/python-${nativeVersion}${architecture}${extension}"
 
         return $uri
     }

config/python-manifest-config.json

@@ -0,0 +1,8 @@
+{
+    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+)",
+    "groups": {
+        "arch": 4,
+        "platform": 2,
+        "platform_version": 3
+    }
+}

helpers/azure-devops/azure-devops-api.ps1

@@ -1,89 +0,0 @@
-class AzureDevOpsApi
-{
-    [string] $BaseUrl
-    [string] $RepoOwner
-    [object] $AuthHeader
-
-    AzureDevOpsApi(
-        [string] $TeamFoundationCollectionUri,
-        [string] $ProjectName,
-        [string] $AccessToken
-    ) {
-        $this.BaseUrl = $this.BuildBaseUrl($TeamFoundationCollectionUri, $ProjectName)
-        $this.AuthHeader = $this.BuildAuth($AccessToken)
-    }
-
-    [object] hidden BuildAuth([string]$AccessToken) {
-        if ([string]::IsNullOrEmpty($AccessToken)) {
-            return $null
-        }
-        return @{
-            Authorization = "Bearer $AccessToken"
-        }
-    }
-
-    [string] hidden BuildBaseUrl([string]$TeamFoundationCollectionUri, [string]$ProjectName) {
-        return "${TeamFoundationCollectionUri}/${ProjectName}/_apis"
-    }
-
-    [object] QueueBuild([string]$ToolVersion, [string]$SourceBranch, [string]$SourceVersion, [UInt32]$DefinitionId){
-        $url = "build/builds"
-        
-        # The content of parameters field should be a json string
-        $buildParameters = @{ VERSION = $ToolVersion } | ConvertTo-Json
-
-        $body = @{
-            definition = @{
-                id = $DefinitionId
-            }
-            sourceBranch = $SourceBranch
-            sourceVersion = $SourceVersion
-            parameters = $buildParameters
-        } | ConvertTo-Json
-
-        return $this.InvokeRestMethod($url, 'POST', $body)
-    }
-
-    [object] GetBuildInfo([UInt32]$BuildId){
-        $url = "build/builds/$BuildId"
-
-        return $this.InvokeRestMethod($url, 'GET', $null)
-    }
-
-    [string] hidden BuildUrl([string]$Url) {
-        return "$($this.BaseUrl)/${Url}/?api-version=5.1"
-    }
-
-    [object] hidden InvokeRestMethod(
-        [string] $Url,
-        [string] $Method,
-        [string] $Body
-    ) {
-        $requestUrl = $this.BuildUrl($Url)
-        $params = @{
-            Method = $Method
-            ContentType = "application/json"
-            Uri = $requestUrl
-            Headers = @{}
-        }
-        if ($this.AuthHeader) {
-            $params.Headers += $this.AuthHeader
-        }
-        if (![string]::IsNullOrEmpty($body)) {
-            $params.Body = $Body
-        }
-
-        return Invoke-RestMethod @params
-    }
-
-}
-
-function Get-AzureDevOpsApi {
-    param (
-        [string] $TeamFoundationCollectionUri,
-        [string] $ProjectName,
-        [string] $AccessToken
-    )
-
-    return [AzureDevOpsApi]::New($TeamFoundationCollectionUri, $ProjectName, $AccessToken)
-}

helpers/azure-devops/build-info.ps1

@@ -1,44 +0,0 @@
-Import-Module (Join-Path $PSScriptRoot "azure-devops-api.ps1")
-
-class BuildInfo
-{
-    [AzureDevOpsApi] $AzureDevOpsApi
-    [String] $Name
-    [UInt32] $Id
-    [String] $Status
-    [String] $Result
-    [String] $Link
-
-    BuildInfo([AzureDevOpsApi] $AzureDevOpsApi, [object] $Build)
-    {
-        $this.AzureDevOpsApi = $AzureDevOpsApi
-        $this.Id = $Build.id
-        $this.Name = $Build.buildNumber
-        $this.Link = $Build._links.web.href
-        $this.Status = $Build.status
-        $this.Result = $Build.result
-    }
-
-    [boolean] IsFinished() {
-        return ($this.Status -eq "completed") -or ($this.Status -eq "cancelling")
-    }
-
-    [boolean] IsSuccess() {
-        return $this.Result -eq "succeeded"
-    }
-
-    [void] UpdateBuildInfo() {
-        $buildInfo = $this.AzureDevOpsApi.GetBuildInfo($this.Id)
-        $this.Status = $buildInfo.status
-        $this.Result = $buildInfo.result
-    }
-}
-
-function Get-BuildInfo {
-    param (
-        [AzureDevOpsApi] $AzureDevOpsApi,
-        [object] $Build
-    )
-
-    return [BuildInfo]::New($AzureDevOpsApi, $Build)
-}

helpers/azure-devops/run-ci-builds.ps1

@@ -1,94 +0,0 @@
-param (
-    [Parameter(Mandatory)] [string] $TeamFoundationCollectionUri,
-    [Parameter(Mandatory)] [string] $AzureDevOpsProjectName,
-    [Parameter(Mandatory)] [string] $AzureDevOpsAccessToken,
-    [Parameter(Mandatory)] [string] $SourceBranch,
-    [Parameter(Mandatory)] [string] $ToolVersions,
-    [Parameter(Mandatory)] [UInt32] $DefinitionId,
-    [string] $SourceVersion
-)
-
-Import-Module (Join-Path $PSScriptRoot "azure-devops-api.ps1")
-Import-Module (Join-Path $PSScriptRoot "build-info.ps1")
-
-function Queue-Builds {
-    param (
-        [Parameter(Mandatory)] [AzureDevOpsApi] $AzureDevOpsApi,
-        [Parameter(Mandatory)] [string] $ToolVersions,
-        [Parameter(Mandatory)] [string] $SourceBranch,
-        [Parameter(Mandatory)] [string] $SourceVersion,
-        [Parameter(Mandatory)] [string] $DefinitionId
-    )
-
-    [BuildInfo[]]$queuedBuilds = @()
-
-    $ToolVersions.Split(',') | ForEach-Object { 
-        $version = $_.Trim()
-        Write-Host "Queue build for $version..."
-        $queuedBuild = $AzureDevOpsApi.QueueBuild($version, $SourceBranch, $SourceVersion, $DefinitionId)
-        $buildInfo = Get-BuildInfo -AzureDevOpsApi $AzureDevOpsApi -Build $queuedBuild
-        Write-Host "Queued build: $($buildInfo.Link)"
-        $queuedBuilds += $buildInfo
-    }
-
-    return $queuedBuilds
-}
-
-function Wait-Builds {
-    param (
-        [Parameter(Mandatory)] [BuildInfo[]] $Builds
-    )
-
-    $timeoutBetweenRefreshSec = 30
-    
-    do {
-        # If build is still running - refresh its status
-        foreach($build in $builds) {
-            if (!$build.IsFinished()) {
-                $build.UpdateBuildInfo()
-                
-                if ($build.IsFinished()) {
-                   Write-Host "The $($build.Name) build was completed: $($build.Link)"
-                }
-            }
-        }
-    
-        $runningBuildsCount = ($builds | Where-Object { !$_.IsFinished() }).Length
-
-        Start-Sleep -Seconds $timeoutBetweenRefreshSec
-    } while($runningBuildsCount -gt 0)
-}
-
-function Make-BuildsOutput {
-    param (
-        [Parameter(Mandatory)] [BuildInfo[]] $Builds
-    )
-
-    Write-Host "Builds info:"
-    $builds | Format-Table -AutoSize -Property Name,Id,Status,Result,Link | Out-String -Width 10000
-
-    # Return exit code based on status of builds
-    $failedBuilds = ($builds | Where-Object { !$_.IsSuccess() })
-    if ($failedBuilds.Length -ne 0) {
-        Write-Host "##vso[task.logissue type=error;]Builds failed"
-        $failedBuilds | ForEach-Object -Process { Write-Host "##vso[task.logissue type=error;]Name: $($_.Name); Link: $($_.Link)" }
-        Write-Host "##vso[task.complete result=Failed]"
-    } else {
-        Write-host "##[section] All builds have been passed successfully"
-    }
-}
-
-$azureDevOpsApi = Get-AzureDevOpsApi -TeamFoundationCollectionUri $TeamFoundationCollectionUri `
-                                     -ProjectName $AzureDevOpsProjectName `
-                                     -AccessToken $AzureDevOpsAccessToken
-
-$queuedBuilds = Queue-Builds -AzureDevOpsApi $azureDevOpsApi `
-                             -ToolVersions $ToolVersions `
-                             -SourceBranch $SourceBranch `
-                             -SourceVersion $SourceVersion `
-                             -DefinitionId $DefinitionId
-
-Write-Host "Waiting results of builds ..."
-Wait-Builds -Builds $queuedBuilds
-
-Make-BuildsOutput -Builds $queuedBuilds

helpers/common-helpers.psm1

@@ -1,80 +0,0 @@
-<#
-.SYNOPSIS
-The execute command and print all output to the logs
-#>
-function Execute-Command {
-    [CmdletBinding()]
-    param(
-        [Parameter(Mandatory=$true)][string] $Command
-    )
-
-    Write-Debug "Execute $Command"
-
-    try {
-        Invoke-Expression $Command | ForEach-Object { Write-Host $_ }
-    }
-    catch {
-        Write-Host "Error happened during command execution: $Command"
-        Write-Host "##vso[task.logissue type=error;] $_"
-    }
-}
-
-<#
-.SYNOPSIS
-Download file from url and return local path to file
-#>
-function Download-File {
-    param(
-        [Parameter(Mandatory=$true)]
-        [Uri]$Uri,
-        [Parameter(Mandatory=$true)]
-        [String]$OutputFolder
-    )
-
-    $targetFilename = [IO.Path]::GetFileName($Uri)
-    $targetFilepath = Join-Path $OutputFolder $targetFilename
-
-    Write-Debug "Download source from $Uri to $OutFile"
-    try {
-        (New-Object System.Net.WebClient).DownloadFile($Uri, $targetFilepath)
-        return $targetFilepath
-    } catch {
-        Write-Host "Error during downloading file from '$Uri'"
-        "$_"
-        exit 1
-    }    
-}
-
-<#
-.SYNOPSIS
-Generate file that contains the list of all files in particular directory
-#>
-function New-ToolStructureDump {
-    param(
-        [Parameter(Mandatory=$true)]
-        [String]$ToolPath,
-        [Parameter(Mandatory=$true)]
-        [String]$OutputFolder
-    )
-
-    $outputFile = Join-Path $OutputFolder "tools_structure.txt"
-
-    $folderContent = Get-ChildItem -Path $ToolPath -Recurse | Sort-Object | Select-Object -Property FullName, Length
-    $folderContent | ForEach-Object {
-        $relativePath = $_.FullName.Replace($ToolPath, "");
-        return "${relativePath}"
-    } | Out-File -FilePath $outputFile
-}
-
-<#
-.SYNOPSIS
-Check if it is macOS / Ubuntu platform
-#>
-function IsNixPlatform {
-    param(
-        [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()]
-        [String]$Platform
-    )
-
-    return ($Platform -match "macos") -or ($Platform -match "ubuntu")
-}

helpers/generate-versions-manifest.ps1

@@ -1,158 +0,0 @@
-<#
-.SYNOPSIS
-Generate versions manifest based on repository releases
-
-.DESCRIPTION
-Versions manifest is needed to find the latest assets for particular version of tool
-.PARAMETER GitHubRepositoryOwner
-Required parameter. The organization which tool repository belongs
-.PARAMETER GitHubRepositoryName
-Optional parameter. The name of tool repository
-.PARAMETER GitHubAccessToken
-Required parameter. PAT Token to overcome GitHub API Rate limit
-.PARAMETER OutputFile
-Required parameter. File "*.json" where generated results will be saved
-.PARAMETER PlatformMapFile
-Optional parameter. Path to the json file with platform map
-Structure example:
-{
-    "macos-1014": [
-        {
-            "platform": "darwin",
-            "platform_version": "10.14"
-        }, ...
-    ], ...
-}
-#>
-
-param (
-    [Parameter(Mandatory)] [string] $GitHubRepositoryOwner,
-    [Parameter(Mandatory)] [string] $GitHubRepositoryName,
-    [Parameter(Mandatory)] [string] $GitHubAccessToken,
-    [Parameter(Mandatory)] [string] $OutputFile,
-    [string] $PlatformMapFile
-)
-
-Import-Module (Join-Path $PSScriptRoot "github/github-api.psm1")
-
-if ($PlatformMapFile -and (Test-Path $PlatformMapFile)) {
-    $PlatformMap = Get-Content $PlatformMapFile -Raw | ConvertFrom-Json -AsHashtable
-} else {
-    $PlatformMap = @{}
-}
-
-function Get-FileNameWithoutExtension {
-    param (
-        [Parameter(Mandatory)][string]$Filename
-    )
-
-    if ($Filename.EndsWith(".tar.gz")) {
-        $Filename = [IO.path]::GetFileNameWithoutExtension($Filename)
-    }
-
-    return [IO.path]::GetFileNameWithoutExtension($Filename)
-}
-
-function New-AssetItem {
-    param (
-        [Parameter(Mandatory)][string]$Filename,
-        [Parameter(Mandatory)][string]$DownloadUrl,
-        [Parameter(Mandatory)][string]$Arch,
-        [Parameter(Mandatory)][string]$Platform,
-        [string]$PlatformVersion
-    )
-    $asset = New-Object PSObject
-
-    $asset | Add-Member -Name "filename" -Value $Filename -MemberType NoteProperty
-    $asset | Add-Member -Name "arch" -Value $Arch -MemberType NoteProperty
-    $asset | Add-Member -Name "platform" -Value $Platform -MemberType NoteProperty
-    if ($PlatformVersion) { $asset | Add-Member -Name "platform_version" -Value $PlatformVersion -MemberType NoteProperty }
-    $asset | Add-Member -Name "download_url" -Value $DownloadUrl -MemberType NoteProperty
-
-    return $asset
-}
-
-function Build-AssetsList {
-    param (
-        [AllowEmptyCollection()]
-        [Parameter(Mandatory)][array]$ReleaseAssets
-    )
-
-    
-    $assets = @()
-    foreach($releaseAsset in $ReleaseAssets) {
-        $filename = Get-FileNameWithoutExtension -Filename $releaseAsset.name
-        $parts = $filename.Split("-")
-        $arch = $parts[-1]
-        $buildPlatform = [string]::Join("-", $parts[2..($parts.Length-2)])
-
-        if ($PlatformMap[$buildPlatform]) {
-            $PlatformMap[$buildPlatform] | ForEach-Object {
-                $assets += New-AssetItem -Filename $releaseAsset.name `
-                                         -DownloadUrl $releaseAsset.browser_download_url `
-                                         -Arch $arch `
-                                         -Platform $_.platform `
-                                         -PlatformVersion $_.platform_version
-            }
-
-        } else {
-            $assets += New-AssetItem -Filename $releaseAsset.name `
-                                     -DownloadUrl $releaseAsset.browser_download_url `
-                                     -Arch $arch `
-                                     -Platform $buildPlatform
-        }
-    }
-
-    return $assets
-}
-
-function Get-VersionFromRelease {
-    param (
-        [Parameter(Mandatory)][object]$Release
-    )
-    # Release name can contain additional information after ':' so filter it
-    [string]$releaseName = $Release.name.Split(':')[0]
-    [Version]$version = $null
-    if (![Version]::TryParse($releaseName, [ref]$version)) {
-        throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )"
-    }
-
-    return $version
-}
-
-function Build-VersionsManifest {
-    param (
-        [Parameter(Mandatory)][array]$Releases
-    )
-
-    $Releases = $Releases | Sort-Object -Property "published_at" -Descending
-
-    $versionsHash = @{}
-    foreach ($release in $Releases) {
-        if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) {
-            continue
-        }
-
-        [Version]$version = Get-VersionFromRelease $release
-        $versionKey = $version.ToString()
-
-        if ($versionsHash.ContainsKey($versionKey)) {
-            continue
-        }
-
-        $versionsHash.Add($versionKey, [PSCustomObject]@{
-            version = $versionKey
-            stable = $true
-            release_url = $release.html_url
-            files = Build-AssetsList $release.assets
-        })
-    }
-
-    # Sort versions by descending
-    return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true }
-}
-
-$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken
-$releases = $gitHubApi.GetGitHubReleases()
-$versionIndex = Build-VersionsManifest $releases
-$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force

helpers/github/create-pull-request.ps1

@@ -1,106 +0,0 @@
-<#
-.SYNOPSIS
-Create commit with all unstaged changes in repository and create pull-request
-
-.PARAMETER RepositoryOwner
-Required parameter. The organization which tool repository belongs
-.PARAMETER RepositoryName
-Optional parameter. The name of tool repository
-.PARAMETER AccessToken
-Required parameter. PAT Token to authorize
-.PARAMETER BranchName
-Required parameter. The name of branch where changes will be pushed
-.PARAMETER CommitMessage
-Required parameter. The commit message to push changes
-.PARAMETER PullRequestTitle
-Required parameter. The title of pull-request
-.PARAMETER PullRequestBody
-Required parameter. The description of pull-request
-#>
-param (
-    [Parameter(Mandatory)] [string] $RepositoryOwner,
-    [Parameter(Mandatory)] [string] $RepositoryName,
-    [Parameter(Mandatory)] [string] $AccessToken,
-    [Parameter(Mandatory)] [string] $BranchName,
-    [Parameter(Mandatory)] [string] $CommitMessage,
-    [Parameter(Mandatory)] [string] $PullRequestTitle,
-    [Parameter(Mandatory)] [string] $PullRequestBody
-)
-
-Import-Module (Join-Path $PSScriptRoot "github-api.psm1")
-Import-Module (Join-Path $PSScriptRoot "git.psm1")
-
-function Update-PullRequest {
-    Param (
-        [Parameter(Mandatory=$true)]
-        [object] $GitHubApi,
-        [Parameter(Mandatory=$true)]
-        [string] $Title,
-        [Parameter(Mandatory=$true)]
-        [string] $Body,
-        [Parameter(Mandatory=$true)]
-        [string] $BranchName,
-        [Parameter(Mandatory=$true)]
-        [object] $PullRequest
-    )
-
-    $updatedPullRequest = $GitHubApi.UpdatePullRequest($Title, $Body, $BranchName, $PullRequest.number)
-
-    if (($updatedPullRequest -eq $null) -or ($updatedPullRequest.html_url -eq $null)) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected error occurs while updating pull request."
-        exit 1
-    }
-    Write-host "##[section] Pull request updated: $($updatedPullRequest.html_url)"
-}
-
-function Create-PullRequest {
-    Param (
-        [Parameter(Mandatory=$true)]
-        [object] $GitHubApi,
-        [Parameter(Mandatory=$true)]
-        [string] $Title,
-        [Parameter(Mandatory=$true)]
-        [string] $Body,
-        [Parameter(Mandatory=$true)]
-        [string] $BranchName
-    )
-
-    $createdPullRequest = $GitHubApi.CreateNewPullRequest($Title, $Body, $BranchName)
-
-    if (($createdPullRequest -eq $null) -or ($createdPullRequest.html_url -eq $null)) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected error occurs while creating pull request."
-        exit 1
-    }
-
-    Write-host "##[section] Pull request created: $($createdPullRequest.html_url)"
-}
-
-Write-Host "Configure local git preferences"
-Git-ConfigureUser -Name "Service account" -Email "no-reply@microsoft.com"
-
-Write-Host "Create branch: $BranchName"
-Git-CreateBranch -Name $BranchName
-    
-Write-Host "Create commit"
-Git-CommitAllChanges -Message $CommitMessage
-
-Write-Host "Push branch: $BranchName"
-Git-PushBranch -Name $BranchName -Force $true
-
-$gitHubApi = Get-GitHubApi -AccountName $RepositoryOwner -ProjectName $RepositoryName -AccessToken $AccessToken
-$pullRequest = $gitHubApi.GetPullRequest($BranchName, $RepositoryOwner)
-
-if ($pullRequest.Count -gt 0) {
-    Write-Host "Update pull request"
-    Update-PullRequest -GitHubApi $gitHubApi `
-                       -Title $PullRequestTitle `
-                       -Body $PullRequestBody `
-                       -BranchName $BranchName `
-                       -PullRequest $pullRequest[0]
-} else {
-    Write-Host "Create pull request"
-    Create-PullRequest -GitHubApi $gitHubApi `
-                       -Title $PullRequestTitle `
-                       -Body $PullRequestBody `
-                       -BranchName $BranchName
-}

helpers/github/git.psm1

@@ -1,81 +0,0 @@
-<#
-.SYNOPSIS
-Configure git credentials to use with commits
-#>
-function Git-ConfigureUser {
-    Param (
-        [Parameter(Mandatory=$true)]
-        [string] $Name,
-        [Parameter(Mandatory=$true)]
-        [string] $Email
-    )
-
-    git config --global user.name $Name | Out-Host
-    git config --global user.email $Email | Out-Host
-
-    if ($LASTEXITCODE -ne 0) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while configuring git preferences."
-        exit 1
-    }
-}
-
-<#
-.SYNOPSIS
-Create new branch
-#>
-function Git-CreateBranch {
-    Param (
-        [Parameter(Mandatory=$true)]
-        [string] $Name
-    )
-
-    git checkout -b $Name | Out-Host
-
-    if ($LASTEXITCODE -ne 0) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while creating new branch: $Name."
-        exit 1
-    }
-}
-
-<#
-.SYNOPSIS
-Commit all staged and unstaged changes
-#>
-function Git-CommitAllChanges {
-    Param (
-        [Parameter(Mandatory=$true)]
-        [string] $Message
-    )
-
-    git add -A | Out-Host
-    git commit -m "$Message" | Out-Host
-
-    if ($LASTEXITCODE -ne 0) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while commiting changes."
-        exit 1
-    }
-}
-
-<#
-.SYNOPSIS
-Push branch to remote repository
-#>
-function Git-PushBranch {
-    Param (
-        [Parameter(Mandatory=$true)]
-        [string] $Name,
-        [Parameter(Mandatory=$true)]
-        [boolean] $Force
-    )
-
-    if ($Force) {
-        git push --set-upstream origin $Name --force | Out-Host
-    } else {
-        git push --set-upstream origin $Name | Out-Host
-    }
-    
-    if ($LASTEXITCODE -ne 0) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while pushing changes."
-        exit 1
-    }
-}

helpers/github/github-api.psm1

@@ -1,109 +0,0 @@
-<#
-.SYNOPSIS
-The module that contains a bunch of methods to interact with GitHub API V3
-#>
-class GitHubApi
-{
-    [string] $BaseUrl
-    [string] $RepoOwner
-    [object] $AuthHeader
-
-    GitHubApi(
-        [string] $AccountName,
-        [string] $ProjectName,
-        [string] $AccessToken
-    ) {
-        $this.BaseUrl = $this.BuildBaseUrl($AccountName, $ProjectName)
-        $this.AuthHeader = $this.BuildAuth($AccessToken)
-    }
-
-    [object] hidden BuildAuth([string]$AccessToken) {
-        if ([string]::IsNullOrEmpty($AccessToken)) {
-            return $null
-        }
-        $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("'':${AccessToken}"))
-        return @{
-            Authorization = "Basic ${base64AuthInfo}"
-        }
-    }
-
-    [string] hidden BuildBaseUrl([string]$RepositoryOwner, [string]$RepositoryName) {
-        return "https://api.github.com/repos/$RepositoryOwner/$RepositoryName"
-    }
-
-    [object] CreateNewPullRequest([string]$Title, [string]$Body, [string]$BranchName){
-        $requestBody = @{
-            title = $Title
-            body = $Body
-            head = $BranchName
-            base = "master"
-        } | ConvertTo-Json
-
-        $url = "pulls"
-        return $this.InvokeRestMethod($url, 'Post', $null, $requestBody)
-    }
-
-    [object] GetPullRequest([string]$BranchName, [string]$RepositoryOwner){
-        $url = "pulls"
-        return $this.InvokeRestMethod($url, 'GET', "head=${RepositoryOwner}:$BranchName&base=master", $null)
-    }
-
-    [object] UpdatePullRequest([string]$Title, [string]$Body, [string]$BranchName, [string]$PullRequestNumber){
-        $requestBody = @{
-            title = $Title
-            body = $Body
-            head = $BranchName
-            base = "master"
-        } | ConvertTo-Json
-
-        $url = "pulls/$PullRequestNumber"
-        return $this.InvokeRestMethod($url, 'Post', $null, $requestBody)
-    }
-
-    [object] GetGitHubReleases(){
-        $url = "releases"
-        return $this.InvokeRestMethod($url, 'GET', $null, $null)
-    }
-
-    [string] hidden BuildUrl([string]$Url, [string]$RequestParams) {
-        if ([string]::IsNullOrEmpty($RequestParams)) {
-            return "$($this.BaseUrl)/$($Url)"
-        } else {
-            return "$($this.BaseUrl)/$($Url)?$($RequestParams)"
-        }
-    }
-
-    [object] hidden InvokeRestMethod(
-        [string] $Url,
-        [string] $Method,
-        [string] $RequestParams,
-        [string] $Body
-    ) {
-        $requestUrl = $this.BuildUrl($Url, $RequestParams)
-        $params = @{
-            Method = $Method
-            ContentType = "application/json"
-            Uri = $requestUrl
-            Headers = @{}
-        }
-        if ($this.AuthHeader) {
-            $params.Headers += $this.AuthHeader
-        }
-        if (![string]::IsNullOrEmpty($Body)) {
-            $params.Body = $Body
-        }
-
-        return Invoke-RestMethod @params
-    }
-
-}
-
-function Get-GitHubApi {
-    param (
-        [string] $AccountName,
-        [string] $ProjectName,
-        [string] $AccessToken
-    )
-
-    return [GitHubApi]::New($AccountName, $ProjectName, $AccessToken)
-}

helpers/nix-helpers.psm1

@@ -1,50 +0,0 @@
-<#
-.SYNOPSIS
-Pack folder to *.zip format
-#>
-function Pack-Zip {
-    param(
-        [Parameter(Mandatory=$true)]
-        [String]$PathToArchive,
-        [Parameter(Mandatory=$true)]
-        [String]$ToolZipFile
-    )
-
-    Write-Debug "Pack $PathToArchive to $ToolZipFile"
-    Push-Location -Path $PathToArchive
-    zip -q -r $ToolZipFile * | Out-Null
-    Pop-Location
-}
-
-<#
-.SYNOPSIS
-Unpack *.tar file
-#>
-function Extract-TarArchive {
-    param(
-        [Parameter(Mandatory=$true)]
-        [String]$ArchivePath,
-        [Parameter(Mandatory=$true)]
-        [String]$OutputDirectory
-    )
-
-    Write-Debug "tar -C $OutputDirectory -xzf $ArchivePath --strip 1"
-    tar -C $OutputDirectory -xzf $ArchivePath --strip 1
-}
-
-function Create-TarArchive {
-    param(
-        [Parameter(Mandatory=$true)]
-        [String]$SourceFolder,
-        [Parameter(Mandatory=$true)]
-        [String]$ArchivePath,
-        [string]$CompressionType = "gz"
-    )
-
-    $CompressionTypeArgument = If ([string]::IsNullOrWhiteSpace($CompressionType)) { "" } else { "--${CompressionType}" }
-
-    Push-Location $SourceFolder
-    Write-Debug "tar -c $CompressionTypeArgument -f $ArchivePath ."
-    tar -c $CompressionTypeArgument -f $ArchivePath .
-    Pop-Location
-}

helpers/pester-extensions.psm1

@@ -1,33 +0,0 @@
-<#
-.SYNOPSIS
-Pester extension that allows to run command and validate exit code
-.EXAMPLE
-"python file.py" | Should -ReturnZeroExitCode
-#>
-function ShouldReturnZeroExitCode {
-    Param(
-        [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
-        [String]$ActualValue,
-        [switch]$Negate
-    )
-
-    Write-Host "Run command '${ActualValue}'"
-    Invoke-Expression -Command $ActualValue | ForEach-Object { Write-Host $_ }
-    $actualExitCode = $LASTEXITCODE
-
-    [bool]$succeeded = $actualExitCode -eq 0
-    if ($Negate) { $succeeded = -not $succeeded }
-
-    if (-not $succeeded)
-    {
-        $failureMessage = "Command '${ActualValue}' has finished with exit code ${actualExitCode}"
-    }
-
-    return New-Object PSObject -Property @{
-        Succeeded      = $succeeded
-        FailureMessage = $failureMessage
-    }
-}
-
-Add-AssertionOperator -Name ReturnZeroExitCode `
-                    -Test  $function:ShouldReturnZeroExitCode

helpers/win-helpers.psm1

@@ -1,18 +0,0 @@
-function Create-SevenZipArchive {
-    param(
-        [Parameter(Mandatory=$true)]
-        [String]$SourceFolder,
-        [Parameter(Mandatory=$true)]
-        [String]$ArchivePath,
-        [String]$ArchiveType = "zip",
-        [String]$CompressionLevel = 5
-    )
-
-    $ArchiveTypeArgument = "-t${ArchiveType}"
-    $CompressionLevelArgument = "-mx=${CompressionLevel}"
-    
-    Push-Location $SourceFolder
-    Write-Debug "7z a $ArchiveTypeArgument $CompressionLevelArgument $ArchivePath @$SourceFolder"
-    7z a $ArchiveTypeArgument $CompressionLevelArgument $ArchivePath $SourceFolder\*
-    Pop-Location
-}

installers/macos-pkg-setup-template.sh

@@ -0,0 +1,79 @@
+set -e
+
+PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
+PYTHON_PKG_NAME="{{__PKG_NAME__}}"
+ARCH="{{__ARCH__}}"
+MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
+MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
+
+PYTHON_MAJOR=python$MAJOR_VERSION
+PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION
+PYTHON_MAJOR_MINOR=python$MAJOR_VERSION$MINOR_VERSION
+
+if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then
+    # No AGENT_TOOLSDIRECTORY on GitHub images
+    TOOLCACHE_ROOT=$RUNNER_TOOL_CACHE
+else
+    TOOLCACHE_ROOT=$AGENT_TOOLSDIRECTORY
+fi
+
+PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
+PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
+PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
+PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
+
+echo "Check if Python hostedtoolcache folder exist..."
+if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
+    echo "Creating Python hostedtoolcache folder..."
+    mkdir -p $PYTHON_TOOLCACHE_PATH
+else
+    # remove ALL other directories for same major.minor python versions
+    find $PYTHON_TOOLCACHE_PATH -name "${MAJOR_VERSION}.${MINOR_VERSION}.*"|while read python_version;do
+        python_version_arch="$python_version/$ARCH"
+        if [ -e "$python_version_arch" ];then
+            echo "Deleting Python $python_version_arch"
+            rm -rf "$python_version_arch"
+        fi
+    done
+fi
+
+echo "Install Python binaries from prebuilt package"
+sudo installer -pkg $PYTHON_PKG_NAME -target /
+
+echo "Create hostedtoolcach symlinks (Required for the backward compatibility)"
+echo "Create Python $PYTHON_FULL_VERSION folder"
+mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+
+ln -s "${PYTHON_FRAMEWORK_PATH}/bin" bin
+ln -s "${PYTHON_FRAMEWORK_PATH}/include" include
+ln -s "${PYTHON_FRAMEWORK_PATH}/share" share
+ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib
+
+echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
+ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
+
+cd bin/
+
+# This symlink already exists if Python version with the same major.minor version is installed,
+# since we do not remove the framework folder
+if [ ! -f $PYTHON_MAJOR_MINOR ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
+fi
+
+if [ ! -f python ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR python
+fi
+
+chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
+
+echo "Upgrading pip..."
+./python -m ensurepip
+./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location --root-user-action=ignore
+
+echo "Install OpenSSL certificates"
+sh -e "${PYTHON_APPLICATION_PATH}/Install Certificates.command"
+
+echo "Create complete file"
+touch $PYTHON_TOOLCACHE_VERSION_PATH/${ARCH}.complete

installers/map_python_platforms.json

@@ -1,29 +0,0 @@
-{
-    "macos-1014": [
-        {
-            "platform": "darwin",
-            "platform_version": "10.14"
-        },
-        {
-            "platform": "darwin",
-            "platform_version": "10.15"
-        }
-    ],
-    "ubuntu-1604": [
-        {
-            "platform": "linux",
-            "platform_version": "16.04"
-        }
-    ],
-    "ubuntu-1804": [
-        {
-            "platform": "linux",
-            "platform_version": "18.04"
-        }
-    ],
-    "windows-2016": [
-        {
-            "platform": "win32"
-        }
-    ]
-}

installers/nix-setup-template.sh

@@ -1,13 +1,12 @@
 set -e
 
-MAJOR_VERSION="{{__VERSION_MAJOR__}}"
-MINOR_VERSION="{{__VERSION_MINOR__}}"
-BUILD_VERSION="{{__VERSION_BUILD__}}"
+PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
+MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
+MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
 
 PYTHON_MAJOR=python$MAJOR_VERSION
 PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION
 PYTHON_MAJORMINOR=python$MAJOR_VERSION$MINOR_VERSION
-PYTHON_FULL_VERSION=$MAJOR_VERSION.$MINOR_VERSION.$BUILD_VERSION
 
 if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then
     # No AGENT_TOOLSDIRECTORY on GitHub images
@@ -38,7 +37,7 @@ rm $PYTHON_TOOLCACHE_VERSION_ARCH_PATH/setup.sh
 
 cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
 
-echo "Create additional symlinks (Required for UsePythonVersion VSTS task)"
+echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
 ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
 
 cd bin/
@@ -49,9 +48,9 @@ fi
 
 chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR python
 
-echo "Upgrading PIP..."
+echo "Upgrading pip..."
 ./python -m ensurepip
-./python -m pip install --ignore-installed pip
+./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location --root-user-action=ignore
 
 echo "Create complete file"
 touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete

installers/win-setup-template.ps1

@@ -1,10 +1,9 @@
 [String] $Architecture = "{{__ARCHITECTURE__}}"
-[Version] $Version = "{{__VERSION__}}"
+[String] $Version = "{{__VERSION__}}"
 [String] $PythonExecName = "{{__PYTHON_EXEC_NAME__}}"
 
 function Get-RegistryVersionFilter {
-    param
-    (
+    param(
         [Parameter(Mandatory)][String] $Architecture,
         [Parameter(Mandatory)][Int32] $MajorVersion,
         [Parameter(Mandatory)][Int32] $MinorVersion
@@ -12,20 +11,15 @@ function Get-RegistryVersionFilter {
 
     $archFilter = if ($Architecture -eq 'x86') { "32-bit" } else { "64-bit" }
     ### Python 2.7 x86 have no architecture postfix
-    if (($Architecture -eq "x86") -and ($MajorVersion -eq 2))
-    {
+    if (($Architecture -eq "x86") -and ($MajorVersion -eq 2)) {
         "Python $MajorVersion.$MinorVersion.\d+$"
-    }
-    else
-    {
+    } else {
         "Python $MajorVersion.$MinorVersion.*($archFilter)"
     }
 }
 
-function Remove-RegistryEntries
-{
-    param
-    (
+function Remove-RegistryEntries {
+    param(
         [Parameter(Mandatory)][String] $Architecture,
         [Parameter(Mandatory)][Int32] $MajorVersion,
         [Parameter(Mandatory)][Int32] $MinorVersion
@@ -34,18 +28,20 @@ function Remove-RegistryEntries
     $versionFilter = Get-RegistryVersionFilter -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
 
     $regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products"
-    $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName
-    foreach ($key in $regKeys)
-    {
-        if ($key.getValue("DisplayName") -match $versionFilter)
-        {
-            Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose
+    if (Test-Path -Path Registry::$regPath) {
+        $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName
+        foreach ($key in $regKeys) {
+            if ($key.getValue("DisplayName") -match $versionFilter) {
+                Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose
+            }
         }
     }
 
     $regPath = "HKEY_CLASSES_ROOT\Installer\Products"
-    Get-ChildItem -Path Registry::$regPath | Where-Object { $_.GetValue("ProductName") -match $versionFilter } | ForEach-Object {
-        Remove-Item Registry::$_ -Recurse -Force -Verbose
+    if (Test-Path -Path Registry::$regPath) {
+        Get-ChildItem -Path Registry::$regPath | Where-Object { $_.GetValue("ProductName") -match $versionFilter } | ForEach-Object {
+            Remove-Item Registry::$_ -Recurse -Force -Verbose
+        }
     }
 
     $uninstallRegistrySections = @(
@@ -63,40 +59,34 @@ function Remove-RegistryEntries
 }
 
 function Get-ExecParams {
-    param
-    (
+    param(
         [Parameter(Mandatory)][Boolean] $IsMSI,
         [Parameter(Mandatory)][String] $PythonArchPath
     )
 
-    if ($IsMSI) 
-    {
+    if ($IsMSI) {
         "TARGETDIR=$PythonArchPath ALLUSERS=1"
-    } 
-    else 
-    {
+    } else {
         "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1"
     }
 }
 
 $ToolcacheRoot = $env:AGENT_TOOLSDIRECTORY
-if ([string]::IsNullOrEmpty($ToolcacheRoot))
-{
+if ([string]::IsNullOrEmpty($ToolcacheRoot)) {
     # GitHub images don't have `AGENT_TOOLSDIRECTORY` variable
     $ToolcacheRoot = $env:RUNNER_TOOL_CACHE
 }
 $PythonToolcachePath = Join-Path -Path $ToolcacheRoot -ChildPath "Python"
-$PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version.ToString()
+$PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version
 $PythonArchPath = Join-Path -Path $PythonVersionPath -ChildPath $Architecture
 
 $IsMSI = $PythonExecName -match "msi"
 
-$MajorVersion = $Version.Major
-$MinorVersion = $Version.Minor
+$MajorVersion = $Version.Split('.')[0]
+$MinorVersion = $Version.Split('.')[1]
 
 Write-Host "Check if Python hostedtoolcache folder exist..."
-if (-Not (Test-Path $PythonToolcachePath)) 
-{
+if (-Not (Test-Path $PythonToolcachePath)) {
     Write-Host "Create Python toolcache folder"
     New-Item -ItemType Directory -Path $PythonToolcachePath | Out-Null
 }
@@ -104,22 +94,19 @@ if (-Not (Test-Path $PythonToolcachePath))
 Write-Host "Check if current Python version is installed..."
 $InstalledVersions = Get-Item "$PythonToolcachePath\$MajorVersion.$MinorVersion.*\$Architecture"
 
-if ($null -ne $InstalledVersions)
-{
+if ($null -ne $InstalledVersions) {
     Write-Host "Python$MajorVersion.$MinorVersion ($Architecture) was found in $PythonToolcachePath..."
 
-    foreach ($InstalledVersion in $InstalledVersions)
-    {
-        if (Test-Path -Path $InstalledVersion)
-        {
+    foreach ($InstalledVersion in $InstalledVersions) {
+        if (Test-Path -Path $InstalledVersion) {
             Write-Host "Deleting $InstalledVersion..."
             Remove-Item -Path $InstalledVersion -Recurse -Force
-            Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose
+            if (Test-Path -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete") {
+                Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose
+            }
         }
     }
-} 
-else 
-{
+} else {
     Write-Host "No Python$MajorVersion.$MinorVersion.* found"
 }
 
@@ -136,14 +123,18 @@ Write-Host "Install Python $Version in $PythonToolcachePath..."
 $ExecParams = Get-ExecParams -IsMSI $IsMSI -PythonArchPath $PythonArchPath
 
 cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet"
-if ($LASTEXITCODE -ne 0)
-{
+if ($LASTEXITCODE -ne 0) {
     Throw "Error happened during Python installation"
 }
 
+Write-Host "Create `python3` symlink"
+if ($MajorVersion -ne "2") {
+    New-Item -Path "$PythonArchPath\python3.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python.exe"
+}
+
 Write-Host "Install and upgrade Pip"
 $PythonExePath = Join-Path -Path $PythonArchPath -ChildPath "python.exe"
-cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip"
+cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip --no-warn-script-location --root-user-action=ignore"
 
 Write-Host "Create complete file"
 New-Item -ItemType File -Path $PythonVersionPath -Name "$Architecture.complete" | Out-Null

tests/ManifestConfig.Tests.ps1

@@ -0,0 +1,42 @@
+Import-Module (Join-Path $PSScriptRoot "../helpers/packages-generation/manifest-utils.psm1")
+
+$ConfigurationFile = Join-Path $PSScriptRoot "../config/python-manifest-config.json"
+$Configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
+
+$stableTestCases = @(
+    @{ ReleaseName = "python-3.8.3-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
+) | ForEach-Object { $_.Configuration = $Configuration; $_ }
+
+$unstableTestCases = @(
+    @{ ReleaseName = "python-3.9.0-alpha.2-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-beta.2-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-rc.4-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-beta.2-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-beta.2-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
+) | ForEach-Object { $_.Configuration = $Configuration; $_ }
+
+Describe "Python manifest config" {
+    Context "Stable versions" {
+        It "<ReleaseName>" -TestCases $stableTestCases {
+            $Release = @{ name = $ReleaseName }
+            $asset = New-AssetItem -ReleaseAsset $Release -Configuration $Configuration
+            $asset.platform | Should -Be $ExpectedResult.platform
+            $asset.platform_version | Should -Be $ExpectedResult.platform_version
+            $asset.arch | Should -Be $ExpectedResult.arch
+        }
+    }
+
+    Context "Prerelease versions" {
+        It "<ReleaseName>" -TestCases $unstableTestCases {
+            $Release = @{ name = $ReleaseName }
+            $asset = New-AssetItem -ReleaseAsset $Release -Configuration $Configuration
+            $asset.platform | Should -Be $ExpectedResult.platform
+            $asset.platform_version | Should -Be $ExpectedResult.platform_version
+            $asset.arch | Should -Be $ExpectedResult.arch
+        }
+    }
+}

tests/clean-toolcache.ps1

@@ -1,11 +0,0 @@
-if ($env:PLATFORM -match 'windows') {
-  $PythonFilter = "Name like '%Python%'"
-  Get-WmiObject Win32_Product -Filter $PythonFilter | Foreach-Object { 
-      Write-Host "Uninstalling $($_.Name) ..."
-      $_.Uninstall() | Out-Null 
-  }
-}
-
-$PythonToolcachePath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "Python"
-Write-Host "Removing Python toolcache directory ..."
-Remove-Item -Path $PythonToolcachePath -Recurse -Force

tests/python-tests.ps1

@@ -1,31 +1,41 @@
 param (
-    [Version] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    [semver] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
     $Version,
-    [String] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
-    $Platform
+    [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    $Platform,
+    [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    $Architecture
 )
 
 Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
 Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1")
+Import-Module (Join-Path $PSScriptRoot "../builders/python-version.psm1")
 
-function Analyze-MissingModules([string] $buildOutputLocation) {
-    $searchStringStart = "Failed to build these modules:"
-    $searchStringEnd = "running build_scripts"
-    $pattern = "$searchStringStart(.*?)$searchStringEnd"
+BeforeAll {
+    function Analyze-MissingModules([string] $buildOutputLocation) {
+        $searchStringStart = "Failed to build these modules:"
+        $searchStringEnd = "running build_scripts"
+        $pattern = "$searchStringStart(.*?)$searchStringEnd"
 
-    $buildContent = Get-Content -Path $buildOutputLocation
-    $splitBuiltOutput = $buildContent -split "\n";
+        $buildContent = Get-Content -Path $buildOutputLocation
+        $splitBuiltOutput = $buildContent -split "\n";
 
-    ### Search for missing modules that are displayed between the search strings
-    $regexMatch = [regex]::match($SplitBuiltOutput, $Pattern)
-    if ($regexMatch.Success)
-    {
-        Write-Host "Failed missing modules:"
-        Write-Host $regexMatch.Groups[1].Value
-        return 1
+        ### Search for missing modules that are displayed between the search strings
+        $regexMatch = [regex]::match($SplitBuiltOutput, $Pattern)
+        if ($regexMatch.Success)
+        {
+            $module = $regexMatch.Groups[1].Value.Trim()
+            Write-Host "Failed missing modules:"
+            Write-Host $module
+            if ( ($module -eq "_tkinter") -and ( [semver]"$($Version.Major).$($Version.Minor)" -ge [semver]"3.10" -and $Version.PreReleaseLabel ) ) {
+                Write-Host "$module $Version ignored"
+            } else {
+                return 1
+            }
+        }
+
+        return 0
     }
-
-    return 0
 }
 
 Describe "Tests" {
@@ -33,7 +43,7 @@ Describe "Tests" {
         "python --version" | Should -ReturnZeroExitCode
         $pythonLocation = (Get-Command "python").Path
         $pythonLocation | Should -Not -BeNullOrEmpty
-        $expectedPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "Python"
+        $expectedPath = Join-Path -Path $env:RUNNER_TOOL_CACHE -ChildPath "Python"
         $pythonLocation.startsWith($expectedPath) | Should -BeTrue
     }
 
@@ -41,10 +51,28 @@ Describe "Tests" {
         "python ./sources/simple-test.py" | Should -ReturnZeroExitCode
     }
 
+    # linux has no display name and no $DISPLAY environment variable - skip tk test
+    # if (-not (($Platform -match "ubuntu") -or ($Platform -match "linux"))) {
+    #     It "Check if tcl/tk has the same headed and library versions" {
+	#     "python ./sources/tcltk.py" | Should -ReturnZeroExitCode
+    #     }
+    # }
+
+    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($Architecture -ne "arm64"))) {
+        It "Check if sqlite3 module is installed" {
+            "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
+        }
+    }
+
+    It "Run pip" {
+        "pip install requests" | Should -ReturnZeroExitCode
+        "pip uninstall requests -y" | Should -ReturnZeroExitCode
+    }
+
     if (IsNixPlatform $Platform) {
 
         It "Check for failed modules in build_output" {
-            $buildOutputLocation = Join-Path $env:BUILD_BINARIESDIRECTORY "build_output.txt"
+            $buildOutputLocation = Join-Path $env:RUNNER_TEMP "build_output.txt"
             Analyze-MissingModules $buildOutputLocation | Should -Be 0
         }
 
@@ -53,7 +81,8 @@ Describe "Tests" {
         }
 
         It "Check if python configuration is correct" {
-            "python ./sources/python-config-test.py" | Should -ReturnZeroExitCode
+            $nativeVersion = Convert-Version -version $Version
+            "python ./sources/python-config-test.py $Version $nativeVersion $Architecture" | Should -ReturnZeroExitCode
         }
 
         It "Check if shared libraries are linked correctly" {
@@ -62,11 +91,16 @@ Describe "Tests" {
     }
 
     # Pyinstaller 3.5 does not support Python 3.8.0. Check issue https://github.com/pyinstaller/pyinstaller/issues/4311
-    if ($Version -lt "3.8.0") {
+    if ($Version -lt "3.8.0" -and $Version.Major -ne "2") {
         It "Validate Pyinstaller" {
             "pip install pyinstaller" | Should -ReturnZeroExitCode
             "pyinstaller --onefile ./sources/simple-test.py" | Should -ReturnZeroExitCode
-            "./dist/simple-test" | Should -ReturnZeroExitCode
+            $distPath = [IO.Path]::Combine($pwd, "dist", "simple-test")
+            "$distPath" | Should -ReturnZeroExitCode
         }
     }
-}
+
+    It "Check urlopen with HTTPS works" {
+        "python ./sources/python-urlopen-https.py" | Should -ReturnZeroExitCode
+    }
+}

tests/sources/python-config-output.py

@@ -1,6 +1,4 @@
-import distutils.sysconfig
 import sysconfig
 
 from pprint import pprint
 pprint(sysconfig.get_config_vars())
-pprint(distutils.sysconfig.get_config_vars())

tests/sources/python-config-test.py

@@ -1,4 +1,3 @@
-import distutils.sysconfig
 import sysconfig
 import sys
 import platform
@@ -6,7 +5,15 @@ import os
 
 # Define variables
 os_type = platform.system()
-version = sys.version.split(" ")[0]
+version = sys.argv[1]
+nativeVersion = sys.argv[2]
+architecture = sys.argv[3]
+
+versions=version.split(".")
+version_major=int(versions[0])
+version_minor=int(versions[1])
+
+pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (architecture == "arm64"))
 
 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
@@ -17,7 +24,11 @@ have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE")
 ### Define expected variables
 if os_type == 'Linux': expected_ld_library_extension = 'so'
 if os_type == 'Darwin': expected_ld_library_extension = 'dylib'
-expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version)
+
+if pkg_installer:
+    expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib'
+else:
+    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/x64/lib'
 
 # Check modules
 ### Validate libraries path
@@ -28,7 +39,7 @@ if lib_dir_path != expected_lib_dir_path:
 ### Validate shared libraries
 if is_shared:
     print('%s was built with shared extensions' % ld_library_name)
-    
+
     ### Validate libpython extension
     ld_library_extension = ld_library_name.split('.')[-1]
     if ld_library_extension != expected_ld_library_extension:
@@ -36,12 +47,13 @@ if is_shared:
         exit(1)
 else:
     print('%s was built without shared extensions' % ld_library_name)
-    exit(1)
+    if not pkg_installer:
+        exit(1)
 
 ### Validate macOS
 if os_type == 'Darwin':
     ### Validate openssl links
-    if version < "3.7.0":
+    if version_major == 3 and version_minor < 7:
         expected_ldflags = '-L/usr/local/opt/openssl@1.1/lib'
         ldflags = sysconfig.get_config_var('LDFLAGS')
 
@@ -49,20 +61,23 @@ if os_type == 'Darwin':
             print('Invalid ldflags: %s; Expected: %s' % (ldflags, expected_ldflags))
             exit(1)
     else:
-        expected_openssl_includes = '-I/usr/local/opt/openssl/include'
-        expected_openssl_ldflags ='-L/usr/local/opt/openssl/lib'
-        
+        expected_openssl_includes = '-I/usr/local/opt/openssl@1.1/include'
+        expected_openssl_ldflags ='-L/usr/local/opt/openssl@1.1/lib'
+
         openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES')
         openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS')
 
         if openssl_includes != expected_openssl_includes:
             print('Invalid openssl_includes: %s; Expected: %s' % (openssl_includes, expected_openssl_includes))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
         if openssl_ldflags != expected_openssl_ldflags:
             print('Invalid openssl_ldflags: %s; Expected: %s' % (openssl_ldflags, expected_openssl_ldflags))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
 
 ### Validate libreadline
-if not have_libreadline:
-    print('Missing libreadline')
-    exit(1)
+if sys.version_info < (3, 12):
+    if not have_libreadline:
+        print('Missing libreadline')
+        exit(1)

tests/sources/python-modules.py

@@ -139,6 +139,7 @@ standard_library = [
     'sre_constants',
     'sre_parse',
     'ssl',
+    '_ssl',
     'stat',
     'string',
     'stringprep',
@@ -251,6 +252,28 @@ if sys.version_info >= (3, 7):
 if sys.version_info > (3, 7):
     standard_library.remove('macpath')
 
+# 'dummy_threading' module has been removed from Python 3.9
+if sys.version_info > (3, 8):
+    standard_library.remove('dummy_threading')
+
+# 'symbol' and 'formatter' modules have been removed from Python 3.10
+if sys.version_info >= (3, 10):
+    standard_library.remove('symbol')
+    standard_library.remove('formatter')
+
+# 'binhex' module has been removed from Python 3.11
+if sys.version_info >= (3, 11):
+    standard_library.remove('binhex')
+
+# 'smtpd', 'asyncore' and 'asynchat' modules have been removed from Python 3.12
+# https://docs.python.org/dev/whatsnew/3.12.html
+if sys.version_info >= (3, 12):
+    standard_library.remove('distutils')
+    standard_library.remove('imp')
+    standard_library.remove('smtpd')
+    standard_library.remove('asyncore')
+    standard_library.remove('asynchat')
+
 # Remove tkinter and Easter eggs
 excluded_modules = [
     'antigravity',
@@ -261,7 +284,6 @@ excluded_modules = [
 def check_missing_modules(expected_modules):
     missing = []
     for module in expected_modules:
-        print('Try to import module ', module)
         try:
             importlib.import_module(module)
         except:

tests/sources/python-sqlite3.py

@@ -0,0 +1,19 @@
+import sqlite3
+from sqlite3 import Error
+
+def create_connection(db_file):
+    """ create a database connection to a SQLite database """
+    conn = None
+    try:
+        print('Sqlite3 version: ', sqlite3.version)
+        conn = sqlite3.connect(db_file)
+        conn.enable_load_extension(True)
+    except Error as e:
+        print(e)
+        exit(1)
+    finally:
+        if conn:
+            conn.close()
+
+if __name__ == '__main__':
+    create_connection(r"pythonsqlite.db")

tests/sources/python-urlopen-https.py

@@ -0,0 +1,10 @@
+import sys
+
+if sys.version_info[0] == 2:
+    from urllib2 import urlopen
+else:
+    from urllib.request import urlopen
+
+response = urlopen("https://raw.githubusercontent.com/actions/python-versions/c641695f6a07526c18f10e374e503e649fef9427/.gitignore")
+data = response.read()
+assert len(data) == 140, len(data)

tests/sources/tcltk.py

@@ -0,0 +1,10 @@
+import tkinter
+import _tkinter
+
+header = _tkinter.TK_VERSION
+lib = tkinter.Tk().getvar('tk_version')
+
+if lib != header:
+  print('header version=' + header)
+  print('lib version=' + lib)
+  exit(1)