Update versions-manifest (#240)

Co-authored-by: Service account <no-reply@microsoft.com>

.github/CODEOWNERS

@@ -1 +1 @@
-* @actions/virtual-environments-owners
+* @actions/setup-actions-team

.github/workflows/build-python-packages.yml

@@ -0,0 +1,220 @@
+name: Build Python package
+run-name: Generate Python ${{ inputs.VERSION || '3.11.0' }}
+on:
+  workflow_dispatch:
+    inputs:
+      VERSION:
+        description: 'Python version to build and upload'
+        default: '3.11.0'
+        required: true
+      PUBLISH_RELEASES:
+        description: 'Whether to publish releases'
+        required: true
+        type: boolean
+        default: false
+      PLATFORMS:
+        description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
+        required: true
+        default: 'ubuntu-20.04,ubuntu-22.04,macos-11_x64,macos-11_arm64,windows-2019_x64,windows-2019_x86'
+  pull_request:
+    paths-ignore:
+    - 'versions-manifest.json'
+    - 'LICENSE'
+    - '**.md'
+    branches:
+    - 'main'
+
+env:
+  VERSION: ${{ inputs.VERSION || '3.11.0' }}
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  generate_matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generate-matrix.outputs.matrix }}
+    steps:
+      - name: Generate execution matrix
+        id: generate-matrix
+        run: |
+          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,macos-11,macos-11_arm64,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
+          $matrix = @()
+
+          foreach ($configuration in $configurations) {
+            $parts = $configuration.Split("_")
+            $os = $parts[0]
+            $arch = if ($parts[1]) {$parts[1]} else {"x64"}
+            switch -wildcard ($os) {
+              "*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
+              "*macos*" { $platform = 'darwin' }
+              "*windows*" { $platform = 'win32' }
+            }
+            $matrix += @{
+              'platform' = $platform
+              'os' = $os
+              'arch' = $arch
+            }
+          }
+          echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT
+
+  build_python:
+    needs: generate_matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
+    runs-on: ${{ matrix.os }}
+    env:
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
+    steps:
+
+      - name: Check out repository code
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Build Python ${{ env.VERSION }}
+        run: |
+          ./builders/build-python.ps1 -Version $env:VERSION `
+                  -Platform ${{ matrix.platform }} -Architecture ${{ matrix.arch }}
+
+      - name: Publish artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ${{ runner.temp }}/artifact
+
+  test_python:
+    needs: [generate_matrix, build_python]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
+    runs-on: ${{ matrix.os }}
+    env:
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
+    steps:
+
+      - name: Check out repository code
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Fully cleanup the toolcache directory before testing
+        run: ./helpers/clean-toolcache.ps1 -ToolName "Python"
+
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ runner.temp }}
+
+      - name: Extract files
+        run: |
+          if ('${{ matrix.platform }}' -eq 'win32') {
+            $artifactName = "${{ env.ARTIFACT_NAME }}.zip"
+            7z.exe x "$artifactName" -y | Out-Null
+          } else {
+            $artifactName = "${{ env.ARTIFACT_NAME }}.tar.gz"
+            tar -xzf $artifactName
+          }
+        working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
+
+      - name: Apply build artifact to the local machine
+        run: |
+          if ('${{ matrix.platform }}' -eq 'win32') {
+            powershell ./setup.ps1
+            } else {
+              sh ./setup.sh
+              cp ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}/build_output.txt ${{ runner.temp }}
+            }
+        working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
+
+      - name: Setup Python ${{ env.VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.VERSION }}
+          architecture: ${{ matrix.arch }}
+
+      - name: Verbose sysconfig dump
+        if: runner.os == 'Linux' || runner.os == 'macOS'
+        run: python ./sources/python-config-output.py
+        working-directory: ${{ github.workspace }}/tests
+
+      - name: Verbose python binary links
+        if: runner.os == 'Linux' || runner.os == 'macOS'
+        run: |
+          $pythonLocation = which python
+          if ('${{ matrix.platform }}' -eq 'darwin') { otool -L $pythonLocation } else { ldd $pythonLocation }
+
+      - name: Run tests
+        run: |
+            Install-Module Pester -Force -Scope CurrentUser
+            Import-Module Pester
+            $pesterContainer = New-PesterContainer -Path './python-tests.ps1' -Data @{
+                Version="${{ env.VERSION }}";
+                Platform="${{ matrix.platform }}";
+                Architecture="${{ matrix.arch }}";
+            }
+            $Result = Invoke-Pester -Container $pesterContainer -PassThru
+            if ($Result.FailedCount -gt 0) {
+              $host.SetShouldExit($Result.FailedCount)
+              exit $Result.FailedCount
+            }
+        working-directory: ${{ github.workspace }}/tests
+
+  publish_release:
+      name: Publish release
+      if: github.event_name == 'workflow_dispatch' && inputs.PUBLISH_RELEASES
+      needs: test_python
+      runs-on: ubuntu-latest
+      steps:
+      - uses: actions/download-artifact@v3
+
+      - name: Publish Release ${{ env.VERSION }}
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.VERSION }}-${{ github.run_id }}
+          release_name: ${{ env.VERSION }}
+          body: |
+            Python ${{ env.VERSION }}
+
+      - name: Upload release assets
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            for (let artifactDir of fs.readdirSync('.')) {
+              let artifactName = fs.readdirSync(`${artifactDir}`)[0];
+              console.log(`Upload ${artifactName} asset`);
+              github.rest.repos.uploadReleaseAsset({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                release_id: ${{ steps.create_release.outputs.id }},
+                name: artifactName,
+                data: fs.readFileSync(`./${artifactDir}/${artifactName}`)
+              });
+            }
+
+  trigger_pr:
+    name: Trigger "Create Pull Request" workflow
+    needs: publish_release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Trigger "Create Pull Request" workflow
+      uses: actions/github-script@v6
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          github.rest.actions.createWorkflowDispatch({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            workflow_id: 'create-pr.yml',
+            ref: 'main'
+          });
+

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,16 @@
+name: CodeQL analysis
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 3 * * 0'
+
+jobs:
+  call-codeQL-analysis:
+    name: CodeQL analysis 
+    uses: actions/reusable-workflows/.github/workflows/codeql-analysis.yml@main
+    with:
+     languages: "['python']"

.github/workflows/create-pr.yml

@@ -2,32 +2,9 @@ name: Create Pull Request
 on:
   workflow_dispatch:
 
-defaults:
-  run:
-    shell: pwsh
-
 jobs:
-  create_pr:
-    name: Create Pull Request
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        submodules: true
-
-    - name: Create versions-manifest.json
-      run: |
-        ./helpers/packages-generation/manifest-generator.ps1 -RepositoryFullName "$env:GITHUB_REPOSITORY" `
-                                                             -GitHubAccessToken "${{secrets.GITHUB_TOKEN}}" `
-                                                             -OutputFile "./versions-manifest.json" `
-                                                             -ConfigurationFile "./config/python-manifest-config.json"
-    - name: Create GitHub PR
-      run: |
-        $formattedDate = Get-Date -Format "MM/dd/yyyy"
-        ./helpers/github/create-pull-request.ps1 `
-                            -RepositoryFullName "$env:GITHUB_REPOSITORY" `
-                            -AccessToken "${{secrets.GITHUB_TOKEN}}" `
-                            -BranchName "update-versions-manifest-file" `
-                            -CommitMessage "Update versions-manifest" `
-                            -PullRequestTitle "[versions-manifest] Update for release from ${formattedDate}" `
-                            -PullRequestBody "Update versions-manifest.json for release from ${formattedDate}"
+  create-pr:
+    uses: actions/versions-package-tools/.github/workflows/create-pr-to-update-manifest.yml@main
+    with:
+      tool-name: "python"
+    secrets: inherit

.github/workflows/get-python-versions.yml

@@ -0,0 +1,13 @@
+name: Get Python versions
+on:
+  schedule:
+    - cron: '0 3,15 * * *'
+  workflow_dispatch:
+
+jobs:
+  get-new-python-versions:
+    uses: actions/versions-package-tools/.github/workflows/get-new-tool-versions.yml@main
+    with:
+      tool-name: "Python"
+      image-url: "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+    secrets: inherit

.github/workflows/manifest-config-validation.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         submodules: true
 
@@ -21,7 +21,7 @@ jobs:
         Install-Module Pester -Force -Scope CurrentUser
 
     - name: Run tests
-      shell: pwsh  
+      shell: pwsh
       run: |
         Import-Module Pester
         Invoke-Pester -Configuration @{

.github/workflows/python-versions-runner.yml

@@ -0,0 +1,31 @@
+name: Python versions runner
+on:
+  workflow_dispatch:
+    inputs:
+      versions:
+        description: 'Versions to build'
+        required: true
+        default: '","'
+      publish-releases:
+        description: 'Whether to publish releases'
+        required: true
+        default: 'false'
+
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  trigger_builds:
+    name: Trigger python build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      
+      - name: Trigger python workflow
+        run: |
+          $versions = ${{ github.event.inputs.versions }}
+          ./builders/python-versions-runner.ps1 -Versions $versions.Split(",") -PublishRelease ${{ github.event.inputs.publish-releases }}
+        env:
+          TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/releases-validation.yml

@@ -1,5 +1,6 @@
 name: Validate 'versions-manifest.json' file
-on: 
+on:
+  workflow_dispatch:
   push:
     branches:
       - main
@@ -13,11 +14,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-16.04, ubuntu-18.04, ubuntu-20.04]
-        python: [3.5.4, 3.6.7, 3.7.5, 3.8.1]
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04]
+        python: [3.9.12, 3.10.8, 3.11.10]
     steps:
     - name: setup-python ${{ matrix.python }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python }}
 
@@ -32,4 +33,4 @@ jobs:
       shell: pwsh
 
     - name: Run simple code
-      run: python -c 'import math; print(math.factorial(5))'
+      run: python -c 'import math; print(math.factorial(5))'

.github/workflows/test-python-version.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-18.04, ubuntu-20.04] 
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04] 
     steps:
     - name: Setup Python ${{ github.event.inputs.version }}
       uses: actions/setup-python@main

.github/workflows/validate-manifest.yml

@@ -0,0 +1,21 @@
+name: Validate manifest
+on:
+# The GITHUB_TOKEN secret is used to create a PR
+# The pull_request event will not be triggered by it
+# That's one of the reasons we need the schedule to validate the versions-manifest.json file
+  schedule:
+    - cron: '0 8,20 * * *'
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'versions-manifest.json'
+
+jobs:
+  manifest:
+    uses: actions/versions-package-tools/.github/workflows/validate-manifest.yml@main
+    with:
+      tool-name: "Python"
+      image-url: "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+    secrets: inherit

README.md

@@ -1,16 +1,25 @@
 # Python for Actions
-This repository contains the code and scripts that we use to build Python packages used in [virtual-environments](https://github.com/actions/virtual-environments) and accessible through the [setup-python](https://github.com/actions/setup-python) Action.  
-File [versions-manifest.json](./versions-manifest.json) contains the list of available and released versions.  
+This repository contains the code and scripts that we use to build Python packages used in [runner-images](https://github.com/actions/runner-images) and accessible through the [setup-python](https://github.com/actions/setup-python) Action.
+File [versions-manifest.json](./versions-manifest.json) contains the list of available and released versions.
 
-> Caution: this is prepared for and only permitted for use by actions `virtual-environments` and `setup-python` action.
+> Caution: this is prepared for and only permitted for use by actions `runner-images` and `setup-python` action.
 
-**Status**: Currently under development and in use for beta and preview actions.  This repo is undergoing rapid changes.
+Some versions are pre-installed on [runner-images](https://github.com/actions/runner-images) images.
+More versions are available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
 
-Some versions are pre-installed on [virtual-environments](https://github.com/actions/virtual-environments) images.  
-More versions will (soon!) be available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
+## Building installation packages
+**Ubuntu:** The official Python sources are downloaded from [python.org](https://www.python.org/ftp/python/), built using the `make` tool, and archived along with the installation script for further distribution and installation. We build each Python version against all [versions of Ubuntu](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) that are available at the time the Python version is released.
 
-## Adding new versions
-We are trying to build and release new versions of Python as soon as they are released. Please open an issue in [actions/virtual-environments](https://github.com/actions/virtual-environments) if any versions are missing.
+**macOS:** 
+- For **Python < 3.11**, the official Python sources are downloaded from [python.org](https://www.python.org/ftp/python/), built using the `make` tool, and archived along with the installation script for further distribution and installation.  For backward compatibility, we build each Python version against the oldest [version of macOS](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) that is available at the time the Python version is released.
+- For **Python >= 3.11**, the official macOS `universal2` Python binaries are simply downloaded from [python.org](https://www.python.org/ftp/python/) and archived along with the installation script for further distribution and installation.
+
+**Windows:** The official Python executables are simply downloaded from [python.org](https://www.python.org/ftp/python/) and archived along with the installation script for further distribution and installation.
+
+## Support policy
+We aim to make new versions of Python available as soon as they are released. Both stable and prerelease versions are considered for building and releasing. Please open an issue in [actions/setup-python](https://github.com/actions/setup-python) if a version you're looking for isn't available.
+
+When a new version of an operating system is released and made available for use with [GitHub hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources), we will provide the **latest existing patch versions of Python for all major versions that have not reached [end-of-life](https://devguide.python.org/versions/)**.  Any subsequent Python versions will be made available for the new OS as well.
 
 ## Contribution
 Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure

azure-pipelines/build-python-packages.yml

@@ -1,132 +0,0 @@
-name: $(date:yyyyMMdd)$(rev:.r)-Python-$(VERSION)
-trigger: none
-pr: none
-
-stages:
-- stage: Build_Python_MacOS
-  dependsOn: []
-  variables:
-    VmImage: 'macOS-10.14'
-    Platform: darwin
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_MacOS
-  condition: succeeded()
-  dependsOn: Build_Python_MacOS
-  variables:
-    VmImage: 'macOS-10.14'
-    Platform: darwin
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_Ubuntu_1604
-  dependsOn: []
-  variables:
-    VmImage: 'ubuntu-16.04'
-    Platform: linux-16.04
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_Ubuntu_1604
-  condition: succeeded()
-  dependsOn: Build_Python_Ubuntu_1604
-  variables:
-    VmImage: 'ubuntu-16.04'
-    Platform: linux-16.04
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_Ubuntu_1804
-  dependsOn: []
-  variables:
-    VmImage: 'ubuntu-18.04'
-    Platform: linux-18.04
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_Ubuntu_1804
-  condition: succeeded()
-  dependsOn: Build_Python_Ubuntu_1804
-  variables:
-    VmImage: 'ubuntu-18.04'
-    Platform: linux-18.04
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_Ubuntu_2004
-  dependsOn: []
-  variables:
-    VmImage: 'ubuntu-20.04'
-    Platform: linux-20.04
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_Ubuntu_2004
-  condition: succeeded()
-  dependsOn: Build_Python_Ubuntu_2004
-  variables:
-    VmImage: 'ubuntu-20.04'
-    Platform: linux-20.04
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_X64_Windows
-  dependsOn: []
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: win32
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_x64_Windows
-  condition: succeeded()
-  dependsOn: Build_Python_X64_Windows
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: win32
-    Architecture: x64
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Build_Python_x86_Windows
-  dependsOn: []
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: win32
-    Architecture: x86
-  jobs:
-  - template: /azure-pipelines/templates/build-job.yml
-
-- stage: Test_Python_x86_Windows
-  condition: succeeded()
-  dependsOn: Build_Python_x86_Windows
-  variables:
-    VmImage: 'vs2017-win2016'
-    Platform: win32
-    Architecture: x86
-  jobs:
-  - template: /azure-pipelines/templates/test-job.yml
-
-- stage: Publish_Release
-  dependsOn: [Test_Python_MacOS, Test_Python_Ubuntu_1604, Test_Python_Ubuntu_1804, Test_Python_Ubuntu_2004, Test_Python_x64_Windows, Test_Python_x86_Windows]
-  jobs:
-  - deployment: Publish_Release
-    pool:
-      name: Azure Pipelines
-      vmImage: ubuntu-18.04
-    environment: 'Get Available Tools Versions - Publishing Approval'
-    strategy:
-      runOnce:
-        deploy:
-          steps:
-          - template: /azure-pipelines/templates/publish-release-steps.yml

azure-pipelines/run-ci-builds.yml

@@ -1,38 +0,0 @@
-trigger: none
-pr:
-  autoCancel: true
-  branches:
-    include:
-    - main
-  paths:
-    exclude:
-    - versions-manifest.json
-
-jobs:
-- job: Run_Builds
-  pool:
-    name: Azure Pipelines
-    vmImage: 'ubuntu-latest'
-  timeoutInMinutes: 180
-
-  steps:
-  - checkout: self
-    submodules: true
-
-  - task: PowerShell@2
-    displayName: 'Run build'
-    inputs:
-      targetType: filePath
-      filePath: './helpers/azure-devops/run-ci-builds.ps1 '
-      arguments: |
-          -TeamFoundationCollectionUri $(System.TeamFoundationCollectionUri) `
-          -AzureDevOpsProjectName $(System.TeamProject) `
-          -AzureDevOpsAccessToken $(System.AccessToken) `
-          -SourceBranch $(Build.SourceBranch) `
-          -DefinitionId $(DEFINITION_ID) `
-          -SourceVersion $(Build.SourceVersion) `
-          -ManifestLink $(MANIFEST_LINK) `
-          -WaitForBuilds $(WAIT_FOR_BUILDS) `
-          -ToolVersions "$(PYTHON_VERSIONS)" `
-          -RetryIntervalSec $(RETRY_INTERVAL_SEC) `
-          -RetryCount $(RETRY_COUNT)

azure-pipelines/templates/build-job.yml

@@ -1,23 +0,0 @@
-jobs:
-- job: Build_Python
-  timeoutInMinutes: 90
-  pool:
-    name: Azure Pipelines
-    vmImage: $(VmImage)
-  steps:
-  - checkout: self
-    submodules: true
-
-  - task: PowerShell@2
-    displayName: 'Build Python $(VERSION)'
-    inputs:
-      targetType: filePath
-      filePath: './builders/build-python.ps1'
-      arguments: '-Version $(VERSION) -Platform $(Platform) -Architecture $(Architecture)'
-      pwsh: true
-
-  - task: PublishPipelineArtifact@1
-    displayName: 'Publish Artifact: Python $(VERSION)'
-    inputs:
-      targetPath: '$(Build.ArtifactStagingDirectory)' 
-      artifactName: 'python-$(VERSION)-$(Platform)-$(Architecture)'

azure-pipelines/templates/publish-release-steps.yml

@@ -1,47 +0,0 @@
-steps:
-- download: none
-
-- checkout: self
-  submodules: true
-
-- task: DownloadPipelineArtifact@2
-  inputs:
-    source: 'current'
-    path: $(Build.BinariesDirectory)
-
-- task: PowerShell@2
-  displayName: 'Create release Python $(VERSION)'
-  inputs:
-    TargetType: inline
-    script: |
-      $tagName = "$(VERSION)-$(Build.BuildId)"
-      $releaseBody = "Python $(VERSION)"
-      ./helpers/github/create-release.ps1 -RepositoryFullName "$(Build.Repository.Name)" `
-                                          -AccessToken "$(GITHUB_TOKEN)" `
-                                          -ToolVersion "$(VERSION)" `
-                                          -TagName "$tagName" `
-                                          -ReleaseBody "$releaseBody" `
-                                          -EventType "$(EVENT_TYPE)"
-- task: GitHubRelease@1
-  displayName: 'Upload release assets'
-  inputs:
-    gitHubConnection: 'Github Connection'
-    action: edit
-    tag: '$(VERSION)-$(Build.BuildId)'
-    title: '$(VERSION)'
-    releaseNotesSource: inline
-    releaseNotesInline: '$(RELEASE_NOTES_CONTENT)'
-    assets: '$(Build.BinariesDirectory)/*/*'
-    assetUploadMode: replace
-    addChangeLog: false
-
-- task: PowerShell@2
-  displayName: 'Trigger "Create Pull Request" workflow'
-  inputs:
-    TargetType: inline
-    script: |
-      Import-Module (Join-Path (Get-Location).Path "github-api.psm1")
-      $gitHubApi = Get-GitHubApi -RepositoryFullName "$(Build.Repository.Name)" -AccessToken "$(GITHUB_TOKEN)"
-      $gitHubApi.CreateWorkflowDispatch("$(WORKFLOW_FILE_NAME)", "$(WORKFLOW_DISPATCH_REF)", "$(INPUTS)")
-      Write-Host "Please find created Pull request here: $(Build.Repository.Uri)/pulls"
-    workingDirectory: '$(Build.SourcesDirectory)/helpers/github'

azure-pipelines/templates/test-job.yml

@@ -1,91 +0,0 @@
-jobs:
-- job: Test_Python
-  pool: 
-    name: Azure Pipelines
-    vmImage: $(VmImage)
-  variables:
-    TestRunTitle: 'python-$(Platform)-$(Architecture)'
-  steps:
-  - checkout: self
-    submodules: true
-
-  - task: PowerShell@2
-    displayName: Fully cleanup the toolcache directory before testing
-    inputs:
-      targetType: filePath
-      filePath: helpers/clean-toolcache.ps1
-      arguments: -ToolName "Python"
-
-  - task: DownloadPipelineArtifact@2
-    inputs:
-      source: 'current'
-      artifact: 'python-$(VERSION)-$(Platform)-$(Architecture)'
-      path: $(Build.BinariesDirectory)
-
-  - task: ExtractFiles@1
-    inputs:
-      archiveFilePatterns: '$(Build.BinariesDirectory)/python-$(VERSION)-$(Platform)-$(Architecture).*'
-      destinationFolder: $(Build.BinariesDirectory)
-      cleanDestinationFolder: false
-      overwriteExistingFiles: true
-
-  - task: PowerShell@2
-    displayName: 'Apply build artifact to the local machines'
-    inputs:
-      TargetType: inline
-      script: |
-        if ($env:PLATFORM -match 'win32') { powershell ./setup.ps1 } else { sh ./setup.sh }
-      workingDirectory: '$(Build.BinariesDirectory)'
-
-  - task: UsePythonVersion@0
-    displayName: 'Use Python $(VERSION)'
-    inputs:
-      versionSpec: '$(VERSION)'
-      architecture: '$(Architecture)'
-
-  - task: PowerShell@2
-    displayName: 'Verbose sysconfig dump'
-    inputs:
-      TargetType: inline
-      script: |
-        Invoke-Expression "python ./sources/python-config-output.py"
-      workingDirectory: '$(Build.SourcesDirectory)/tests'
-    condition: ne(variables['Platform'], 'win32')
-
-  - task: PowerShell@2
-    displayName: 'Verbose python binary links'
-    inputs:
-      TargetType: inline
-      script: |
-        $pythonLocation = which python
-        if ($env:PLATFORM -match 'darwin') { otool -L $pythonLocation } else { ldd $pythonLocation }
-      workingDirectory: '$(Build.BinariesDirectory)'
-    condition: ne(variables['Platform'], 'win32')
-
-  - task: PowerShell@2
-    displayName: 'Run tests'
-    inputs:
-      TargetType: inline
-      script: |
-        Install-Module Pester -Force -Scope CurrentUser -RequiredVersion 4.10.1
-        Import-Module Pester
-        $pesterParams = @{
-          Path="./python-tests.ps1";
-          Parameters=@{
-            Version="$(VERSION)";
-            Platform="$(Platform)";
-          }
-        }
-        Invoke-Pester -Script $pesterParams -OutputFile "test_results.xml" -OutputFormat NUnitXml
-      pwsh: true
-      workingDirectory: '$(Build.SourcesDirectory)/tests'
-
-  - task: PublishTestResults@2
-    displayName: 'Publish test results'
-    inputs:
-      testResultsFiles: '*.xml'
-      testResultsFormat: NUnit
-      searchFolder: 'tests'
-      failTaskOnFailedTests: true
-      testRunTitle: "$(TestRunTitle)"
-    condition: always()

builders/invoke-workflow.psm1

@@ -0,0 +1,52 @@
+function Invoke-Workflow {
+    param (
+        [string] $Version,
+        [string] $PublishRelease
+    )
+    
+    $payload = @{
+        "ref" = "main"
+        "inputs" = @{
+            "VERSION" = "$Version"
+            "PUBLISH_RELEASES" = "$PublishRelease"
+        }
+    } | ConvertTo-Json
+    $headers = @{
+        Authorization="Bearer $env:TOKEN"
+    }
+    $actionsRepoUri = "$env:GITHUB_API_URL/repos/$env:GITHUB_REPOSITORY/actions"
+    Invoke-RestMethod -uri "$actionsRepoUri/workflows/build-python-packages.yml/dispatches" -method POST -headers $headers -body $payload
+
+    $result = [PSCustomObject]@{
+        Version = $Version
+        Conclusion = "failure"
+        Url = "Not run"
+    }
+    # Triggering workflow and verifying that it has been triggered with retries
+    while (-not $workflowToCheck) {
+        Start-Sleep -seconds 40
+        $workflowRuns = (Invoke-RestMethod "$actionsRepoUri/runs").workflow_runs | Where-Object {$_.status -like "*progress*" -and $_.id -ne $env:GITHUB_RUN_ID}
+        $workflowToCheck = $workflowRuns | Where-Object {
+            (Invoke-RestMethod "$actionsRepoUri/runs/$($_.id)/jobs").jobs.steps.name -like "*$Version"
+        }
+        $retries++
+        if ($retries -gt 10) {
+            Write-Host "Workflow triggered for version '$Version' not found or something went wrong with fetching the workflow status"
+            return $result
+        }
+    }
+    # Waiting for workflow to complete
+    while ($workflowToCheck.status -ne "completed") {
+        Start-Sleep -Seconds 120
+        $workflowToCheck = Invoke-RestMethod "$actionsRepoUri/runs/$($workflowToCheck.id)"
+        Write-Host "Workflow run with Id: $($workflowToCheck.id) for version '$Version' - status '$($workflowToCheck.status)'"
+    }
+    $result.Conclusion = $workflowToCheck.conclusion
+    $result.Url = $workflowToCheck.html_url
+    if ($workflowToCheck.conclusion -ne "success") {
+        Write-Host "Triggered workflow for version '$Version' completed unsuccessfully with result '$($workflowToCheck.conclusion)'. Check the logs: $($workflowToCheck.html_url)"
+        return $result
+    }
+    Write-Host "Triggered workflow for version '$Version' succeeded; Url: $($workflowToCheck.html_url)"
+    return $result
+}

builders/macos-python-builder.psm1

@@ -8,6 +8,10 @@ class macOSPythonBuilder : NixPythonBuilder {
     .DESCRIPTION
     Contains methods that required to build macOS Python artifact from sources. Inherited from base NixPythonBuilder.
 
+    While python.org provides precompiled binaries for macOS, switching to them risks breaking existing customers.
+    If we wanted to start using the official binaries instead of building from source, we should avoid changing previous versions
+    so we remain backwards compatible.
+
     .PARAMETER platform
     The full name of platform for which Python should be built.
 
@@ -22,6 +26,21 @@ class macOSPythonBuilder : NixPythonBuilder {
         [string] $platform
     ) : Base($version, $architecture, $platform) { }
 
+    [void] PrepareEnvironment() {
+        <#
+        .SYNOPSIS
+        Prepare system environment by installing dependencies and required packages.
+        #>
+        
+        if ($this.Version -eq "3.7.17") {
+            # We have preinstalled ncurses and readLine on the hoster runners. But we need to install bzip2 for 
+            # setting up an environemnt 
+            # If we get any issues realted to ncurses or readline we can try to run this command
+            # brew install ncurses readline
+            Execute-Command -Command "brew install bzip2"
+        }
+    }
+
     [void] Configure() {
         <#
         .SYNOPSIS
@@ -35,15 +54,40 @@ class macOSPythonBuilder : NixPythonBuilder {
         $configureString += " --enable-shared"
         $configureString += " --with-lto"
 
+        ### For Python versions which support it, compile a universal2 (arm64 + x86_64 hybrid) build. The arm64 slice
+        ### will never be used itself by a Github Actions runner but using a universal2 Python is the only way to build
+        ### universal2 C extensions and wheels. This is supported by Python >= 3.10 and was backported to Python >=
+        ### 3.9.1 and >= 3.8.10.
+        ### Disabled, discussion: https://github.com/actions/python-versions/pull/114
+        # if ($this.Version -ge "3.8.10" -and $this.Version -ne "3.8.13" -and $this.Version -ne "3.9.0" ) {
+        #     $configureString += " --enable-universalsdk --with-universal-archs=universal2"
+        # }
+
         ### OS X 10.11, Apple no longer provides header files for the deprecated system version of OpenSSL.
         ### Solution is to install these libraries from a third-party package manager,
         ### and then add the appropriate paths for the header and library files to configure command.
         ### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies)
         if ($this.Version -lt "3.7.0") {
-            $env:LDFLAGS = "-L/usr/local/opt/openssl@1.1/lib"
-            $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include"
+            $env:LDFLAGS = "-L/usr/local/opt/openssl@1.1/lib -L/usr/local/opt/zlib/lib"
+            $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/zlib/include"
         } else {
             $configureString += " --with-openssl=/usr/local/opt/openssl@1.1"
+
+            # For Python 3.7.2 and 3.7.3 we need to provide PATH for zlib to pack it properly. Otherwise the build will fail
+            # with the error: zipimport.ZipImportError: can't decompress data; zlib not available
+            if ($this.Version -eq "3.7.2" -or $this.Version -eq "3.7.3" -or $this.Version -eq "3.7.17") {
+                $env:LDFLAGS = "-L/usr/local/opt/zlib/lib"
+                $env:CFLAGS = "-I/usr/local/opt/zlib/include"
+            }
+
+            if ($this.Version -gt "3.7.12") {
+                $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'"
+	        }
+
+            if ($this.Version -eq "3.7.17") {
+                $env:LDFLAGS += " -L$(brew --prefix bzip2)/lib -L$(brew --prefix readline)/lib -L$(brew --prefix ncurses)/lib"
+                $env:CFLAGS += " -I$(brew --prefix bzip2)/include -I$(brew --prefix readline)/include -I$(brew --prefix ncurses)/include"
+            }
         }
 
         ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
@@ -52,19 +96,101 @@ class macOSPythonBuilder : NixPythonBuilder {
             $configureString += " --enable-loadable-sqlite-extensions"
             $env:LDFLAGS += " -L$(brew --prefix sqlite3)/lib"
             $env:CFLAGS += " -I$(brew --prefix sqlite3)/include"
+            $env:CPPFLAGS += "-I$(brew --prefix sqlite3)/include"
         }
 
+        Write-Host "The passed configure options are: "
+        Write-Host $configureString
+
         Execute-Command -Command $configureString
     }
 
-    [void] PrepareEnvironment() {
+    [string] GetPkgName() {
         <#
         .SYNOPSIS
-        Prepare system environment by installing dependencies and required packages.
+        Return Python installation Package.
         #>
 
-        ### reinstall header files to Avoid issue with X11 headers on Mojave
-        $pkgName = "/Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg"
-        Execute-Command -Command "sudo installer -pkg $pkgName -target /"
+        $nativeVersion = Convert-Version -version $this.Version
+        $architecture = "-macos11"
+        $extension = ".pkg"
+
+        $pkg = "python-${nativeVersion}${architecture}${extension}"
+
+        return $pkg
+    }
+
+    [uri] GetPkgUri() {
+        <#
+        .SYNOPSIS
+        Get base Python URI and return complete URI for Python installation package.
+        #>
+
+        $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $pkg = $this.GetPkgName()
+
+        $uri = "${base}/${versionName}/${pkg}"
+
+        return $uri
+    }
+
+    [string] DownloadPkg() {
+        <#
+        .SYNOPSIS
+        Download Python installation executable into artifact location.
+        #>
+
+        $pkgUri = $this.GetPkgUri()
+
+        Write-Host "Sources URI: $pkgUri"
+        $pkgLocation = Download-File -Uri $pkgUri -OutputFolder $this.WorkFolderLocation
+        Write-Debug "Done; Package location: $pkgLocation"
+
+        New-Item -Path $this.WorkFolderLocation -Name "build_output.txt"  -ItemType File
+        return $pkgLocation
+    }
+
+    [void] CreateInstallationScriptPkg() {
+        <#
+        .SYNOPSIS
+        Create Python artifact installation script based on specified template.
+        #>
+
+        $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath "macos-pkg-setup-template.sh"
+        $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
+        $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName  -ItemType File
+
+        $variablesToReplace = @{
+            "{{__VERSION_FULL__}}" = $this.Version;
+            "{{__PKG_NAME__}}" = $this.GetPkgName();
+            "{{__ARCH__}}" = $this.Architecture;
+        }
+
+        $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
+        $installationTemplateContent | Out-File -FilePath $installationScriptLocation
+        Write-Debug "Done; Installation script location: $installationScriptLocation)"
+    }
+
+    [void] Build() {
+        <#
+        .SYNOPSIS
+        Generates Python artifact from downloaded Python installation executable.
+        #>
+
+        $PkgVersion = [semver]"3.11.0-beta.1"
+
+        if (($this.Version -ge $PkgVersion) -or ($this.Architecture -eq "arm64")) {
+            Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..."
+            $this.DownloadPkg()
+
+            Write-Host "Create installation script..."
+            $this.CreateInstallationScriptPkg()
+        } else {
+            ([NixPythonBuilder]$this).Build()
+        }
+
+        Write-Host "Archive artifact"
+        $this.ArchiveArtifact()
     }
 }

builders/python-builder.psm1

@@ -41,10 +41,16 @@ class PythonBuilder {
     PythonBuilder ([semver] $version, [string] $architecture, [string] $platform) {
         $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers"
 
-        $this.HostedToolcacheLocation = $env:AGENT_TOOLSDIRECTORY
-        $this.TempFolderLocation = $env:BUILD_SOURCESDIRECTORY
-        $this.WorkFolderLocation = $env:BUILD_BINARIESDIRECTORY
-        $this.ArtifactFolderLocation = $env:BUILD_STAGINGDIRECTORY
+        $artifactDirectory = Join-Path $env:RUNNER_TEMP "artifact"
+        $workDirectory = Join-Path $env:RUNNER_TEMP "work"
+
+        New-Item -Force -Type Directory $artifactDirectory
+        New-Item -Force -Type Directory $workDirectory
+
+        $this.HostedToolcacheLocation = $env:RUNNER_TOOL_CACHE
+        $this.TempFolderLocation = $env:RUNNER_TEMP
+        $this.WorkFolderLocation =  $workDirectory
+        $this.ArtifactFolderLocation = $artifactDirectory
 
         $this.Version = $version
         $this.Architecture = $architecture

builders/python-versions-runner.ps1

@@ -0,0 +1,31 @@
+<#
+.SYNOPSIS
+Generate Python artifact.
+
+.DESCRIPTION
+Script that triggering and fetching the result of the "Build python package" workflows with provided python versions
+
+.PARAMETER Version
+Required parameter. Python versions to trigger builds for.
+
+.PARAMETER PublishRelease
+Switch parameter. Whether to publish release for built version.
+
+#>
+
+param(
+    [Parameter (Mandatory=$true, HelpMessage="Python version to trigger build for")]
+    [array] $Versions,
+    [Parameter (Mandatory=$false, HelpMessage="Whether to publish release for built version")]
+    [string] $PublishRelease
+)
+
+$summary = $Versions | ForEach-Object -Parallel { 
+    Import-Module "./builders/invoke-workflow.psm1"
+    Invoke-Workflow -Version $_ -PublishRelease $Using:PublishRelease
+}
+Write-Host "Results of triggered workflows:"
+$summary | Out-String
+if ($summary.Conclusion -contains "failure" -or $summary.Conclusion -contains "cancelled") {
+    exit 1
+}

builders/ubuntu-python-builder.psm1

@@ -48,6 +48,9 @@ class UbuntuPythonBuilder : NixPythonBuilder {
             $configureString += " --enable-loadable-sqlite-extensions"
         }
 
+        Write-Host "The passed configure options are: "
+        Write-Host $configureString
+
         Execute-Command -Command $configureString
     }
 
@@ -69,6 +72,7 @@ class UbuntuPythonBuilder : NixPythonBuilder {
             $tkinterInstallString = "sudo apt install -y python-tk tk-dev"
         }
 
+        Execute-Command -Command "sudo apt-get update"
         Execute-Command -Command $tkinterInstallString
 
         ### Install dependent packages
@@ -86,10 +90,7 @@ class UbuntuPythonBuilder : NixPythonBuilder {
             Execute-Command -Command "sudo apt install -y $_"
         }
 
-        if ($this.Platform -ne "linux-16.04") {
-            ### On Ubuntu-1804, libgdbm-compat-dev has older modules that are no longer in libgdbm-dev
-            Execute-Command -Command "sudo apt install -y libgdbm-compat-dev"
-        }
-        
+        ### On Ubuntu-1804, libgdbm-compat-dev has older modules that are no longer in libgdbm-dev
+        Execute-Command -Command "sudo apt install -y libgdbm-compat-dev"
     }
 }

config/python-manifest-config.json

@@ -1,5 +1,5 @@
 {
-    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?(x\\d+)",
+    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+)",
     "groups": {
         "arch": 4,
         "platform": 2,

installers/macos-pkg-setup-template.sh

@@ -0,0 +1,79 @@
+set -e
+
+PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
+PYTHON_PKG_NAME="{{__PKG_NAME__}}"
+ARCH="{{__ARCH__}}"
+MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
+MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
+
+PYTHON_MAJOR=python$MAJOR_VERSION
+PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION
+PYTHON_MAJOR_MINOR=python$MAJOR_VERSION$MINOR_VERSION
+
+if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then
+    # No AGENT_TOOLSDIRECTORY on GitHub images
+    TOOLCACHE_ROOT=$RUNNER_TOOL_CACHE
+else
+    TOOLCACHE_ROOT=$AGENT_TOOLSDIRECTORY
+fi
+
+PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
+PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
+PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
+PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
+
+echo "Check if Python hostedtoolcache folder exist..."
+if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
+    echo "Creating Python hostedtoolcache folder..."
+    mkdir -p $PYTHON_TOOLCACHE_PATH
+else
+    # remove ALL other directories for same major.minor python versions
+    find $PYTHON_TOOLCACHE_PATH -name "${MAJOR_VERSION}.${MINOR_VERSION}.*"|while read python_version;do
+        python_version_arch="$python_version/$ARCH"
+        if [ -e "$python_version_arch" ];then
+            echo "Deleting Python $python_version_arch"
+            rm -rf "$python_version_arch"
+        fi
+    done
+fi
+
+echo "Install Python binaries from prebuilt package"
+sudo installer -pkg $PYTHON_PKG_NAME -target /
+
+echo "Create hostedtoolcach symlinks (Required for the backward compatibility)"
+echo "Create Python $PYTHON_FULL_VERSION folder"
+mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+
+ln -s "${PYTHON_FRAMEWORK_PATH}/bin" bin
+ln -s "${PYTHON_FRAMEWORK_PATH}/include" include
+ln -s "${PYTHON_FRAMEWORK_PATH}/share" share
+ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib
+
+echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
+ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
+
+cd bin/
+
+# This symlink already exists if Python version with the same major.minor version is installed,
+# since we do not remove the framework folder
+if [ ! -f $PYTHON_MAJOR_MINOR ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
+fi
+
+if [ ! -f python ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR python
+fi
+
+chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
+
+echo "Upgrading pip..."
+./python -m ensurepip
+./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location --root-user-action=ignore
+
+echo "Install OpenSSL certificates"
+sh -e "${PYTHON_APPLICATION_PATH}/Install Certificates.command"
+
+echo "Create complete file"
+touch $PYTHON_TOOLCACHE_VERSION_PATH/${ARCH}.complete

installers/nix-setup-template.sh

@@ -48,9 +48,9 @@ fi
 
 chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR python
 
-echo "Upgrading PIP..."
+echo "Upgrading pip..."
 ./python -m ensurepip
-./python -m pip install --ignore-installed pip --no-warn-script-location
+./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location --root-user-action=ignore
 
 echo "Create complete file"
 touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete

installers/win-setup-template.ps1

@@ -28,16 +28,20 @@ function Remove-RegistryEntries {
     $versionFilter = Get-RegistryVersionFilter -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
 
     $regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products"
-    $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName
-    foreach ($key in $regKeys) {
-        if ($key.getValue("DisplayName") -match $versionFilter) {
-            Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose
+    if (Test-Path -Path Registry::$regPath) {
+        $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName
+        foreach ($key in $regKeys) {
+            if ($key.getValue("DisplayName") -match $versionFilter) {
+                Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose
+            }
         }
     }
 
     $regPath = "HKEY_CLASSES_ROOT\Installer\Products"
-    Get-ChildItem -Path Registry::$regPath | Where-Object { $_.GetValue("ProductName") -match $versionFilter } | ForEach-Object {
-        Remove-Item Registry::$_ -Recurse -Force -Verbose
+    if (Test-Path -Path Registry::$regPath) {
+        Get-ChildItem -Path Registry::$regPath | Where-Object { $_.GetValue("ProductName") -match $versionFilter } | ForEach-Object {
+            Remove-Item Registry::$_ -Recurse -Force -Verbose
+        }
     }
 
     $uninstallRegistrySections = @(
@@ -97,7 +101,9 @@ if ($null -ne $InstalledVersions) {
         if (Test-Path -Path $InstalledVersion) {
             Write-Host "Deleting $InstalledVersion..."
             Remove-Item -Path $InstalledVersion -Recurse -Force
-            Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose
+            if (Test-Path -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete") {
+                Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose
+            }
         }
     }
 } else {
@@ -128,7 +134,7 @@ if ($MajorVersion -ne "2") {
 
 Write-Host "Install and upgrade Pip"
 $PythonExePath = Join-Path -Path $PythonArchPath -ChildPath "python.exe"
-cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip --no-warn-script-location"
+cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip --no-warn-script-location --root-user-action=ignore"
 
 Write-Host "Create complete file"
 New-Item -ItemType File -Path $PythonVersionPath -Name "$Architecture.complete" | Out-Null

tests/ManifestConfig.Tests.ps1

@@ -5,18 +5,16 @@ $Configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
 
 $stableTestCases = @(
     @{ ReleaseName = "python-3.8.3-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
-    @{ ReleaseName = "python-3.8.3-linux-16.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "16.04"; arch = "x64"} },
-    @{ ReleaseName = "python-3.8.3-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
 ) | ForEach-Object { $_.Configuration = $Configuration; $_ }
 
 $unstableTestCases = @(
     @{ ReleaseName = "python-3.9.0-alpha.2-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
-    @{ ReleaseName = "python-3.9.0-beta.1-linux-16.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "16.04"; arch = "x64"} },
-    @{ ReleaseName = "python-3.9.0-rc.4-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-rc.4-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
 ) | ForEach-Object { $_.Configuration = $Configuration; $_ }

tests/python-tests.ps1

@@ -2,31 +2,40 @@ param (
     [semver] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
     $Version,
     [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
-    $Platform
+    $Platform,
+    [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    $Architecture
 )
 
 Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
 Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1")
 Import-Module (Join-Path $PSScriptRoot "../builders/python-version.psm1")
 
-function Analyze-MissingModules([string] $buildOutputLocation) {
-    $searchStringStart = "Failed to build these modules:"
-    $searchStringEnd = "running build_scripts"
-    $pattern = "$searchStringStart(.*?)$searchStringEnd"
+BeforeAll {
+    function Analyze-MissingModules([string] $buildOutputLocation) {
+        $searchStringStart = "Failed to build these modules:"
+        $searchStringEnd = "running build_scripts"
+        $pattern = "$searchStringStart(.*?)$searchStringEnd"
 
-    $buildContent = Get-Content -Path $buildOutputLocation
-    $splitBuiltOutput = $buildContent -split "\n";
+        $buildContent = Get-Content -Path $buildOutputLocation
+        $splitBuiltOutput = $buildContent -split "\n";
 
-    ### Search for missing modules that are displayed between the search strings
-    $regexMatch = [regex]::match($SplitBuiltOutput, $Pattern)
-    if ($regexMatch.Success)
-    {
-        Write-Host "Failed missing modules:"
-        Write-Host $regexMatch.Groups[1].Value
-        return 1
+        ### Search for missing modules that are displayed between the search strings
+        $regexMatch = [regex]::match($SplitBuiltOutput, $Pattern)
+        if ($regexMatch.Success)
+        {
+            $module = $regexMatch.Groups[1].Value.Trim()
+            Write-Host "Failed missing modules:"
+            Write-Host $module
+            if ( ($module -eq "_tkinter") -and ( [semver]"$($Version.Major).$($Version.Minor)" -ge [semver]"3.10" -and $Version.PreReleaseLabel ) ) {
+                Write-Host "$module $Version ignored"
+            } else {
+                return 1
+            }
+        }
+
+        return 0
     }
-
-    return 0
 }
 
 Describe "Tests" {
@@ -34,7 +43,7 @@ Describe "Tests" {
         "python --version" | Should -ReturnZeroExitCode
         $pythonLocation = (Get-Command "python").Path
         $pythonLocation | Should -Not -BeNullOrEmpty
-        $expectedPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "Python"
+        $expectedPath = Join-Path -Path $env:RUNNER_TOOL_CACHE -ChildPath "Python"
         $pythonLocation.startsWith($expectedPath) | Should -BeTrue
     }
 
@@ -42,7 +51,14 @@ Describe "Tests" {
         "python ./sources/simple-test.py" | Should -ReturnZeroExitCode
     }
 
-    if ($Version -ge "3.2.0") {
+    # linux has no display name and no $DISPLAY environment variable - skip tk test
+    # if (-not (($Platform -match "ubuntu") -or ($Platform -match "linux"))) {
+    #     It "Check if tcl/tk has the same headed and library versions" {
+	#     "python ./sources/tcltk.py" | Should -ReturnZeroExitCode
+    #     }
+    # }
+
+    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($Architecture -ne "arm64"))) {
         It "Check if sqlite3 module is installed" {
             "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
         }
@@ -56,7 +72,7 @@ Describe "Tests" {
     if (IsNixPlatform $Platform) {
 
         It "Check for failed modules in build_output" {
-            $buildOutputLocation = Join-Path $env:BUILD_BINARIESDIRECTORY "build_output.txt"
+            $buildOutputLocation = Join-Path $env:RUNNER_TEMP "build_output.txt"
             Analyze-MissingModules $buildOutputLocation | Should -Be 0
         }
 
@@ -66,7 +82,7 @@ Describe "Tests" {
 
         It "Check if python configuration is correct" {
             $nativeVersion = Convert-Version -version $Version
-            "python ./sources/python-config-test.py $Version $nativeVersion" | Should -ReturnZeroExitCode
+            "python ./sources/python-config-test.py $Version $nativeVersion $Architecture" | Should -ReturnZeroExitCode
         }
 
         It "Check if shared libraries are linked correctly" {
@@ -79,7 +95,12 @@ Describe "Tests" {
         It "Validate Pyinstaller" {
             "pip install pyinstaller" | Should -ReturnZeroExitCode
             "pyinstaller --onefile ./sources/simple-test.py" | Should -ReturnZeroExitCode
-            "./dist/simple-test" | Should -ReturnZeroExitCode
+            $distPath = [IO.Path]::Combine($pwd, "dist", "simple-test")
+            "$distPath" | Should -ReturnZeroExitCode
         }
     }
-}
+
+    It "Check urlopen with HTTPS works" {
+        "python ./sources/python-urlopen-https.py" | Should -ReturnZeroExitCode
+    }
+}

tests/sources/python-config-output.py

@@ -1,6 +1,4 @@
-import distutils.sysconfig
 import sysconfig
 
 from pprint import pprint
 pprint(sysconfig.get_config_vars())
-pprint(distutils.sysconfig.get_config_vars())

tests/sources/python-config-test.py

@@ -1,5 +1,3 @@
-import distutils.sysconfig
-from distutils.version import LooseVersion
 import sysconfig
 import sys
 import platform
@@ -9,6 +7,13 @@ import os
 os_type = platform.system()
 version = sys.argv[1]
 nativeVersion = sys.argv[2]
+architecture = sys.argv[3]
+
+versions=version.split(".")
+version_major=int(versions[0])
+version_minor=int(versions[1])
+
+pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (architecture == "arm64"))
 
 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
@@ -19,7 +24,11 @@ have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE")
 ### Define expected variables
 if os_type == 'Linux': expected_ld_library_extension = 'so'
 if os_type == 'Darwin': expected_ld_library_extension = 'dylib'
-expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version)
+
+if pkg_installer:
+    expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib'
+else:
+    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/x64/lib'
 
 # Check modules
 ### Validate libraries path
@@ -30,7 +39,7 @@ if lib_dir_path != expected_lib_dir_path:
 ### Validate shared libraries
 if is_shared:
     print('%s was built with shared extensions' % ld_library_name)
-    
+
     ### Validate libpython extension
     ld_library_extension = ld_library_name.split('.')[-1]
     if ld_library_extension != expected_ld_library_extension:
@@ -38,12 +47,13 @@ if is_shared:
         exit(1)
 else:
     print('%s was built without shared extensions' % ld_library_name)
-    exit(1)
+    if not pkg_installer:
+        exit(1)
 
 ### Validate macOS
 if os_type == 'Darwin':
     ### Validate openssl links
-    if LooseVersion(nativeVersion) < LooseVersion("3.7.0"):
+    if version_major == 3 and version_minor < 7:
         expected_ldflags = '-L/usr/local/opt/openssl@1.1/lib'
         ldflags = sysconfig.get_config_var('LDFLAGS')
 
@@ -53,18 +63,21 @@ if os_type == 'Darwin':
     else:
         expected_openssl_includes = '-I/usr/local/opt/openssl@1.1/include'
         expected_openssl_ldflags ='-L/usr/local/opt/openssl@1.1/lib'
-        
+
         openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES')
         openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS')
 
         if openssl_includes != expected_openssl_includes:
             print('Invalid openssl_includes: %s; Expected: %s' % (openssl_includes, expected_openssl_includes))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
         if openssl_ldflags != expected_openssl_ldflags:
             print('Invalid openssl_ldflags: %s; Expected: %s' % (openssl_ldflags, expected_openssl_ldflags))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
 
 ### Validate libreadline
-if not have_libreadline:
-    print('Missing libreadline')
-    exit(1)
+if sys.version_info < (3, 12):
+    if not have_libreadline:
+        print('Missing libreadline')
+        exit(1)

tests/sources/python-modules.py

@@ -261,6 +261,19 @@ if sys.version_info >= (3, 10):
     standard_library.remove('symbol')
     standard_library.remove('formatter')
 
+# 'binhex' module has been removed from Python 3.11
+if sys.version_info >= (3, 11):
+    standard_library.remove('binhex')
+
+# 'smtpd', 'asyncore' and 'asynchat' modules have been removed from Python 3.12
+# https://docs.python.org/dev/whatsnew/3.12.html
+if sys.version_info >= (3, 12):
+    standard_library.remove('distutils')
+    standard_library.remove('imp')
+    standard_library.remove('smtpd')
+    standard_library.remove('asyncore')
+    standard_library.remove('asynchat')
+
 # Remove tkinter and Easter eggs
 excluded_modules = [
     'antigravity',

tests/sources/python-urlopen-https.py

@@ -0,0 +1,10 @@
+import sys
+
+if sys.version_info[0] == 2:
+    from urllib2 import urlopen
+else:
+    from urllib.request import urlopen
+
+response = urlopen("https://raw.githubusercontent.com/actions/python-versions/c641695f6a07526c18f10e374e503e649fef9427/.gitignore")
+data = response.read()
+assert len(data) == 140, len(data)

tests/sources/tcltk.py

@@ -0,0 +1,10 @@
+import tkinter
+import _tkinter
+
+header = _tkinter.TK_VERSION
+lib = tkinter.Tk().getvar('tk_version')
+
+if lib != header:
+  print('header version=' + header)
+  print('lib version=' + lib)
+  exit(1)