Update versions-manifest (#242)

Co-authored-by: Service account <no-reply@microsoft.com>

.github/CODEOWNERS

@@ -1 +1 @@
-* @actions/virtual-environments-owners
+* @actions/setup-actions-team

.github/workflows/build-python-packages.yml

@@ -1,20 +1,21 @@
-name: Build python package
-
+name: Build Python package
+run-name: Generate Python ${{ inputs.VERSION || '3.11.0' }}
 on:
   workflow_dispatch:
     inputs:
       VERSION:
         description: 'Python version to build and upload'
-        default: '3.9.9'
+        default: '3.11.0'
         required: true
       PUBLISH_RELEASES:
         description: 'Whether to publish releases'
         required: true
-        default: 'false'
+        type: boolean
+        default: false
       PLATFORMS:
         description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
         required: true
-        default: 'ubuntu-18.04,ubuntu-20.04,ubuntu-22.04,macos-10.15,windows-2019_x64,windows-2019_x86'
+        default: 'ubuntu-20.04,ubuntu-22.04,macos-11_x64,macos-11_arm64,windows-2019_x64,windows-2019_x86'
   pull_request:
     paths-ignore:
     - 'versions-manifest.json'
@@ -24,7 +25,7 @@ on:
     - 'main'
 
 env:
-  VERSION: ${{ github.event.inputs.VERSION || '3.9.9' }}
+  VERSION: ${{ inputs.VERSION || '3.11.0' }}
 defaults:
   run:
     shell: pwsh
@@ -38,7 +39,7 @@ jobs:
       - name: Generate execution matrix
         id: generate-matrix
         run: |
-          $configurations = "${{ github.event.inputs.platforms || 'ubuntu-18.04,ubuntu-20.04,ubuntu-22.04,macos-10.15,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
+          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,macos-11,macos-11_arm64,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
           $matrix = @()
 
           foreach ($configuration in $configurations) {
@@ -56,7 +57,7 @@ jobs:
               'arch' = $arch
             }
           }
-          echo "::set-output name=matrix::$($matrix | ConvertTo-Json -Compress)"
+          echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT
 
   build_python:
     needs: generate_matrix
@@ -66,11 +67,11 @@ jobs:
         include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
     runs-on: ${{ matrix.os }}
     env:
-      ARTIFACT_NAME: python-${{ github.event.inputs.VERSION || '3.9.9' }}-${{ matrix.platform }}-${{ matrix.arch }}
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
     steps:
 
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           submodules: true
 
@@ -80,7 +81,7 @@ jobs:
                   -Platform ${{ matrix.platform }} -Architecture ${{ matrix.arch }}
 
       - name: Publish artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.ARTIFACT_NAME }}
           path: ${{ runner.temp }}/artifact
@@ -93,11 +94,11 @@ jobs:
         include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
     runs-on: ${{ matrix.os }}
     env:
-      ARTIFACT_NAME: python-${{ github.event.inputs.VERSION || '3.9.9' }}-${{ matrix.platform }}-${{ matrix.arch }}
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
     steps:
 
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           submodules: true
 
@@ -105,7 +106,7 @@ jobs:
         run: ./helpers/clean-toolcache.ps1 -ToolName "Python"
 
       - name: Download artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ runner.temp }}
 
@@ -131,7 +132,7 @@ jobs:
         working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
 
       - name: Setup Python ${{ env.VERSION }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ env.VERSION }}
           architecture: ${{ matrix.arch }}
@@ -154,6 +155,7 @@ jobs:
             $pesterContainer = New-PesterContainer -Path './python-tests.ps1' -Data @{
                 Version="${{ env.VERSION }}";
                 Platform="${{ matrix.platform }}";
+                Architecture="${{ matrix.arch }}";
             }
             $Result = Invoke-Pester -Container $pesterContainer -PassThru
             if ($Result.FailedCount -gt 0) {
@@ -164,11 +166,11 @@ jobs:
 
   publish_release:
       name: Publish release
-      if: github.event_name == 'workflow_dispatch' && github.event.inputs.PUBLISH_RELEASES == 'true'
+      if: github.event_name == 'workflow_dispatch' && inputs.PUBLISH_RELEASES
       needs: test_python
       runs-on: ubuntu-latest
       steps:
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
 
       - name: Publish Release ${{ env.VERSION }}
         id: create_release
@@ -182,7 +184,7 @@ jobs:
             Python ${{ env.VERSION }}
 
       - name: Upload release assets
-        uses: actions/github-script@v2
+        uses: actions/github-script@v6
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
@@ -190,7 +192,7 @@ jobs:
             for (let artifactDir of fs.readdirSync('.')) {
               let artifactName = fs.readdirSync(`${artifactDir}`)[0];
               console.log(`Upload ${artifactName} asset`);
-              github.repos.uploadReleaseAsset({
+              github.rest.repos.uploadReleaseAsset({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 release_id: ${{ steps.create_release.outputs.id }},
@@ -205,11 +207,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Trigger "Create Pull Request" workflow
-      uses: actions/github-script@v3
+      uses: actions/github-script@v6
       with:
-        github-token: ${{ secrets.PERSONAL_TOKEN }}
+        github-token: ${{ secrets.GITHUB_TOKEN }}
         script: |
-          github.actions.createWorkflowDispatch({
+          github.rest.actions.createWorkflowDispatch({
             owner: context.repo.owner,
             repo: context.repo.repo,
             workflow_id: 'create-pr.yml',

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,16 @@
+name: CodeQL analysis
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 3 * * 0'
+
+jobs:
+  call-codeQL-analysis:
+    name: CodeQL analysis 
+    uses: actions/reusable-workflows/.github/workflows/codeql-analysis.yml@main
+    with:
+     languages: "['python']"

.github/workflows/create-pr.yml

@@ -2,32 +2,9 @@ name: Create Pull Request
 on:
   workflow_dispatch:
 
-defaults:
-  run:
-    shell: pwsh
-
 jobs:
-  create_pr:
-    name: Create Pull Request
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        submodules: true
-
-    - name: Create versions-manifest.json
-      run: |
-        ./helpers/packages-generation/manifest-generator.ps1 -RepositoryFullName "$env:GITHUB_REPOSITORY" `
-                                                             -GitHubAccessToken "${{secrets.GITHUB_TOKEN}}" `
-                                                             -OutputFile "./versions-manifest.json" `
-                                                             -ConfigurationFile "./config/python-manifest-config.json"
-    - name: Create GitHub PR
-      run: |
-        $formattedDate = Get-Date -Format "MM/dd/yyyy"
-        ./helpers/github/create-pull-request.ps1 `
-                            -RepositoryFullName "$env:GITHUB_REPOSITORY" `
-                            -AccessToken "${{secrets.GITHUB_TOKEN}}" `
-                            -BranchName "update-versions-manifest-file" `
-                            -CommitMessage "Update versions-manifest" `
-                            -PullRequestTitle "[versions-manifest] Update for release from ${formattedDate}" `
-                            -PullRequestBody "Update versions-manifest.json for release from ${formattedDate}"
+  create-pr:
+    uses: actions/versions-package-tools/.github/workflows/create-pr-to-update-manifest.yml@main
+    with:
+      tool-name: "python"
+    secrets: inherit

.github/workflows/get-python-versions.yml

@@ -4,92 +4,10 @@ on:
     - cron: '0 3,15 * * *'
   workflow_dispatch:
 
-env:
-  TOOL_NAME: "Python"
-defaults:
-  run:
-    shell: pwsh
-
 jobs:
-  find_new_versions:
-    name: Find new versions
-    runs-on: ubuntu-latest
-    outputs:
-      versions_output: ${{ steps.Get_new_versions.outputs.TOOL_VERSIONS }}
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - id: Get_new_versions
-        name: Get new versions
-        run: ./helpers/get-new-tool-versions/get-new-tool-versions.ps1 -ToolName ${{ env.TOOL_NAME }}
-
-  check_new_versions:
-    name: Check new versions
-    runs-on: ubuntu-latest
-    needs: find_new_versions
-    env:
-      TOOL_VERSIONS: ${{needs.find_new_versions.outputs.versions_output}}
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - name: Check Versions
-        if: env.TOOL_VERSIONS == ''
-        run: |
-          Write-Host "No new versions were found"
-          Import-Module "./helpers/github/github-api.psm1"
-          $gitHubApi = Get-GitHubApi -RepositoryFullName "$env:GITHUB_REPOSITORY" `
-                                     -AccessToken "${{ secrets.PERSONAL_TOKEN }}"
-          $gitHubApi.CancelWorkflow("$env:GITHUB_RUN_ID")
-          Start-Sleep -Seconds 60
-      - name: Send Slack notification
-        run: |
-          $pipelineUrl = "$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID"
-          $message = "The following versions of '${{ env.TOOL_NAME }}' are available to upload: ${{ env.TOOL_VERSIONS }}\nLink to the pipeline: $pipelineUrl"
-          ./helpers/get-new-tool-versions/send-slack-notification.ps1 -Url "${{ secrets.SLACK_CHANNEL_URL }}" `
-                                                                      -ToolName "${{ env.TOOL_NAME }}" `
-                                                                      -ImageUrl "https://avatars.githubusercontent.com/u/1525981?s=200&v=4" `
-                                                                      -Text "$message"
-  
-  trigger_builds:
-    name: Trigger builds
-    runs-on: ubuntu-latest
-    needs: [find_new_versions, check_new_versions]
-    env:
-      TOOL_VERSIONS: ${{needs.find_new_versions.outputs.versions_output}}
-    environment: Get Available Tools Versions - Publishing Approval
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - name: Trigger "Build python packages" workflow
-        run:
-          ./helpers/github/run-ci-builds.ps1 -RepositoryFullName "$env:GITHUB_REPOSITORY" `
-                                             -AccessToken "${{ secrets.PERSONAL_TOKEN }}" `
-                                             -WorkflowFileName "python-builder.yml" `
-                                             -WorkflowDispatchRef "main" `
-                                             -ToolVersions "${{ env.TOOL_VERSIONS }}" `
-                                             -PublishReleases "true"
-
-  check_build:
-    name: Check build for failures 
-    runs-on: ubuntu-latest
-    needs: [find_new_versions, check_new_versions, trigger_builds]
-    if: failure()
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - name: Send Slack notification if build fails
-        run: |
-          $pipelineUrl = "$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID"
-          $message = "The build of the '${{ env.TOOL_NAME }}' detection pipeline failed :progress-error:\nLink to the pipeline: $pipelineUrl"
-          ./helpers/get-new-tool-versions/send-slack-notification.ps1 -Url "${{ secrets.SLACK_CHANNEL_URL }}" `
-                                                                      -ToolName "${{ env.TOOL_NAME }}" `
-                                                                      -Text "$message" `
-                                                                      -ImageUrl "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+  get-new-python-versions:
+    uses: actions/versions-package-tools/.github/workflows/get-new-tool-versions.yml@main
+    with:
+      tool-name: "Python"
+      image-url: "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+    secrets: inherit

.github/workflows/manifest-config-validation.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         submodules: true
 
@@ -21,7 +21,7 @@ jobs:
         Install-Module Pester -Force -Scope CurrentUser
 
     - name: Run tests
-      shell: pwsh  
+      shell: pwsh
       run: |
         Import-Module Pester
         Invoke-Pester -Configuration @{

.github/workflows/python-versions-runner.yml

@@ -21,11 +21,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       
       - name: Trigger python workflow
         run: |
           $versions = ${{ github.event.inputs.versions }}
           ./builders/python-versions-runner.ps1 -Versions $versions.Split(",") -PublishRelease ${{ github.event.inputs.publish-releases }}
         env:
-          PERSONAL_TOKEN: ${{ secrets.PERSONAL_TOKEN }}
+          TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/releases-validation.yml

@@ -14,11 +14,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-18.04, ubuntu-20.04]
-        python: [3.5.4, 3.6.7, 3.7.5, 3.8.1]
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04]
+        python: [3.9.12, 3.10.8, 3.11.10]
     steps:
     - name: setup-python ${{ matrix.python }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python }}
 

.github/workflows/test-python-version.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-18.04, ubuntu-20.04] 
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04] 
     steps:
     - name: Setup Python ${{ github.event.inputs.version }}
       uses: actions/setup-python@main

.github/workflows/validate-manifest.yml

@@ -1,48 +1,21 @@
 name: Validate manifest
-
 on:
+# The GITHUB_TOKEN secret is used to create a PR
+# The pull_request event will not be triggered by it
+# That's one of the reasons we need the schedule to validate the versions-manifest.json file
   schedule:
     - cron: '0 8,20 * * *'
-
+  workflow_dispatch:
   pull_request:
     branches:
       - main
     paths:
       - 'versions-manifest.json'
 
-env:
-  TOOL_NAME: "Python"
-
-defaults:
-  run:
-    shell: pwsh
-
 jobs:
-  validation:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - name: Validate python-versions manifest
-        run: .\helpers\packages-generation\manifest-validator.ps1 -ManifestPath '.\versions-manifest.json'
-
-  check_build:
-    name: Check validation for failures 
-    runs-on: ubuntu-latest
-    needs: [validation]
-    if: failure()
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - name: Send Slack notification if validation fails
-        run: |
-          $pipelineUrl = "$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID"
-          $message = "The validation of python-versions manifest failed. \nLink to the pipeline: $pipelineUrl"
-          .\helpers\get-new-tool-versions\send-slack-notification.ps1 -Url "${{ secrets.SLACK_CHANNEL_URL }}" `
-                                                                      -ToolName "${{ env.TOOL_NAME }}" `
-                                                                      -Text "$message" `
-                                                                      -ImageUrl "https://www.python.org/static/community_logos/python-powered-h-100x130.png"
+  manifest:
+    uses: actions/versions-package-tools/.github/workflows/validate-manifest.yml@main
+    with:
+      tool-name: "Python"
+      image-url: "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+    secrets: inherit

README.md

@@ -1,16 +1,25 @@
 # Python for Actions
-This repository contains the code and scripts that we use to build Python packages used in [virtual-environments](https://github.com/actions/virtual-environments) and accessible through the [setup-python](https://github.com/actions/setup-python) Action.  
-File [versions-manifest.json](./versions-manifest.json) contains the list of available and released versions.  
+This repository contains the code and scripts that we use to build Python packages used in [runner-images](https://github.com/actions/runner-images) and accessible through the [setup-python](https://github.com/actions/setup-python) Action.
+File [versions-manifest.json](./versions-manifest.json) contains the list of available and released versions.
 
-> Caution: this is prepared for and only permitted for use by actions `virtual-environments` and `setup-python` action.
+> Caution: this is prepared for and only permitted for use by actions `runner-images` and `setup-python` action.
 
-**Status**: Currently under development and in use for beta and preview actions.  This repo is undergoing rapid changes.
+Some versions are pre-installed on [runner-images](https://github.com/actions/runner-images) images.
+More versions are available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
 
-Some versions are pre-installed on [virtual-environments](https://github.com/actions/virtual-environments) images.  
-More versions will (soon!) be available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
+## Building installation packages
+**Ubuntu:** The official Python sources are downloaded from [python.org](https://www.python.org/ftp/python/), built using the `make` tool, and archived along with the installation script for further distribution and installation. We build each Python version against all [versions of Ubuntu](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) that are available at the time the Python version is released.
 
-## Adding new versions
-We are trying to build and release new versions of Python as soon as they are released. Please open an issue in [actions/virtual-environments](https://github.com/actions/virtual-environments) if any versions are missing.
+**macOS:** 
+- For **Python < 3.11**, the official Python sources are downloaded from [python.org](https://www.python.org/ftp/python/), built using the `make` tool, and archived along with the installation script for further distribution and installation.  For backward compatibility, we build each Python version against the oldest [version of macOS](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) that is available at the time the Python version is released.
+- For **Python >= 3.11**, the official macOS `universal2` Python binaries are simply downloaded from [python.org](https://www.python.org/ftp/python/) and archived along with the installation script for further distribution and installation.
+
+**Windows:** The official Python executables are simply downloaded from [python.org](https://www.python.org/ftp/python/) and archived along with the installation script for further distribution and installation.
+
+## Support policy
+We aim to make new versions of Python available as soon as they are released. Both stable and prerelease versions are considered for building and releasing. Please open an issue in [actions/setup-python](https://github.com/actions/setup-python) if a version you're looking for isn't available.
+
+When a new version of an operating system is released and made available for use with [GitHub hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources), we will provide the **latest existing patch versions of Python for all major versions that have not reached [end-of-life](https://devguide.python.org/versions/)**.  Any subsequent Python versions will be made available for the new OS as well.
 
 ## Contribution
 Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure

builders/invoke-workflow.psm1

@@ -12,10 +12,10 @@ function Invoke-Workflow {
         }
     } | ConvertTo-Json
     $headers = @{
-        Authorization="Bearer $env:PERSONAL_TOKEN"
+        Authorization="Bearer $env:TOKEN"
     }
     $actionsRepoUri = "$env:GITHUB_API_URL/repos/$env:GITHUB_REPOSITORY/actions"
-    Invoke-RestMethod -uri "$actionsRepoUri/workflows/python-builder.yml/dispatches" -method POST -headers $headers -body $payload
+    Invoke-RestMethod -uri "$actionsRepoUri/workflows/build-python-packages.yml/dispatches" -method POST -headers $headers -body $payload
 
     $result = [PSCustomObject]@{
         Version = $Version

builders/macos-python-builder.psm1

@@ -8,6 +8,10 @@ class macOSPythonBuilder : NixPythonBuilder {
     .DESCRIPTION
     Contains methods that required to build macOS Python artifact from sources. Inherited from base NixPythonBuilder.
 
+    While python.org provides precompiled binaries for macOS, switching to them risks breaking existing customers.
+    If we wanted to start using the official binaries instead of building from source, we should avoid changing previous versions
+    so we remain backwards compatible.
+
     .PARAMETER platform
     The full name of platform for which Python should be built.
 
@@ -27,6 +31,14 @@ class macOSPythonBuilder : NixPythonBuilder {
         .SYNOPSIS
         Prepare system environment by installing dependencies and required packages.
         #>
+        
+        if ($this.Version -eq "3.7.17") {
+            # We have preinstalled ncurses and readLine on the hoster runners. But we need to install bzip2 for 
+            # setting up an environemnt 
+            # If we get any issues realted to ncurses or readline we can try to run this command
+            # brew install ncurses readline
+            Execute-Command -Command "brew install bzip2"
+        }
     }
 
     [void] Configure() {
@@ -42,6 +54,15 @@ class macOSPythonBuilder : NixPythonBuilder {
         $configureString += " --enable-shared"
         $configureString += " --with-lto"
 
+        ### For Python versions which support it, compile a universal2 (arm64 + x86_64 hybrid) build. The arm64 slice
+        ### will never be used itself by a Github Actions runner but using a universal2 Python is the only way to build
+        ### universal2 C extensions and wheels. This is supported by Python >= 3.10 and was backported to Python >=
+        ### 3.9.1 and >= 3.8.10.
+        ### Disabled, discussion: https://github.com/actions/python-versions/pull/114
+        # if ($this.Version -ge "3.8.10" -and $this.Version -ne "3.8.13" -and $this.Version -ne "3.9.0" ) {
+        #     $configureString += " --enable-universalsdk --with-universal-archs=universal2"
+        # }
+
         ### OS X 10.11, Apple no longer provides header files for the deprecated system version of OpenSSL.
         ### Solution is to install these libraries from a third-party package manager,
         ### and then add the appropriate paths for the header and library files to configure command.
@@ -51,6 +72,22 @@ class macOSPythonBuilder : NixPythonBuilder {
             $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/zlib/include"
         } else {
             $configureString += " --with-openssl=/usr/local/opt/openssl@1.1"
+
+            # For Python 3.7.2 and 3.7.3 we need to provide PATH for zlib to pack it properly. Otherwise the build will fail
+            # with the error: zipimport.ZipImportError: can't decompress data; zlib not available
+            if ($this.Version -eq "3.7.2" -or $this.Version -eq "3.7.3" -or $this.Version -eq "3.7.17") {
+                $env:LDFLAGS = "-L/usr/local/opt/zlib/lib"
+                $env:CFLAGS = "-I/usr/local/opt/zlib/include"
+            }
+
+            if ($this.Version -gt "3.7.12") {
+                $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'"
+	        }
+
+            if ($this.Version -eq "3.7.17") {
+                $env:LDFLAGS += " -L$(brew --prefix bzip2)/lib -L$(brew --prefix readline)/lib -L$(brew --prefix ncurses)/lib"
+                $env:CFLAGS += " -I$(brew --prefix bzip2)/include -I$(brew --prefix readline)/include -I$(brew --prefix ncurses)/include"
+            }
         }
 
         ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
@@ -62,6 +99,98 @@ class macOSPythonBuilder : NixPythonBuilder {
             $env:CPPFLAGS += "-I$(brew --prefix sqlite3)/include"
         }
 
+        Write-Host "The passed configure options are: "
+        Write-Host $configureString
+
         Execute-Command -Command $configureString
     }
+
+    [string] GetPkgName() {
+        <#
+        .SYNOPSIS
+        Return Python installation Package.
+        #>
+
+        $nativeVersion = Convert-Version -version $this.Version
+        $architecture = "-macos11"
+        $extension = ".pkg"
+
+        $pkg = "python-${nativeVersion}${architecture}${extension}"
+
+        return $pkg
+    }
+
+    [uri] GetPkgUri() {
+        <#
+        .SYNOPSIS
+        Get base Python URI and return complete URI for Python installation package.
+        #>
+
+        $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $pkg = $this.GetPkgName()
+
+        $uri = "${base}/${versionName}/${pkg}"
+
+        return $uri
+    }
+
+    [string] DownloadPkg() {
+        <#
+        .SYNOPSIS
+        Download Python installation executable into artifact location.
+        #>
+
+        $pkgUri = $this.GetPkgUri()
+
+        Write-Host "Sources URI: $pkgUri"
+        $pkgLocation = Download-File -Uri $pkgUri -OutputFolder $this.WorkFolderLocation
+        Write-Debug "Done; Package location: $pkgLocation"
+
+        New-Item -Path $this.WorkFolderLocation -Name "build_output.txt"  -ItemType File
+        return $pkgLocation
+    }
+
+    [void] CreateInstallationScriptPkg() {
+        <#
+        .SYNOPSIS
+        Create Python artifact installation script based on specified template.
+        #>
+
+        $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath "macos-pkg-setup-template.sh"
+        $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
+        $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName  -ItemType File
+
+        $variablesToReplace = @{
+            "{{__VERSION_FULL__}}" = $this.Version;
+            "{{__PKG_NAME__}}" = $this.GetPkgName();
+            "{{__ARCH__}}" = $this.Architecture;
+        }
+
+        $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
+        $installationTemplateContent | Out-File -FilePath $installationScriptLocation
+        Write-Debug "Done; Installation script location: $installationScriptLocation)"
+    }
+
+    [void] Build() {
+        <#
+        .SYNOPSIS
+        Generates Python artifact from downloaded Python installation executable.
+        #>
+
+        $PkgVersion = [semver]"3.11.0-beta.1"
+
+        if (($this.Version -ge $PkgVersion) -or ($this.Architecture -eq "arm64")) {
+            Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..."
+            $this.DownloadPkg()
+
+            Write-Host "Create installation script..."
+            $this.CreateInstallationScriptPkg()
+        } else {
+            ([NixPythonBuilder]$this).Build()
+        }
+
+        Write-Host "Archive artifact"
+        $this.ArchiveArtifact()
+    }
 }

builders/ubuntu-python-builder.psm1

@@ -48,6 +48,9 @@ class UbuntuPythonBuilder : NixPythonBuilder {
             $configureString += " --enable-loadable-sqlite-extensions"
         }
 
+        Write-Host "The passed configure options are: "
+        Write-Host $configureString
+
         Execute-Command -Command $configureString
     }
 
@@ -69,6 +72,7 @@ class UbuntuPythonBuilder : NixPythonBuilder {
             $tkinterInstallString = "sudo apt install -y python-tk tk-dev"
         }
 
+        Execute-Command -Command "sudo apt-get update"
         Execute-Command -Command $tkinterInstallString
 
         ### Install dependent packages

config/python-manifest-config.json

@@ -1,5 +1,5 @@
 {
-    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?(x\\d+)",
+    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+)",
     "groups": {
         "arch": 4,
         "platform": 2,

installers/macos-pkg-setup-template.sh

@@ -0,0 +1,79 @@
+set -e
+
+PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
+PYTHON_PKG_NAME="{{__PKG_NAME__}}"
+ARCH="{{__ARCH__}}"
+MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
+MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
+
+PYTHON_MAJOR=python$MAJOR_VERSION
+PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION
+PYTHON_MAJOR_MINOR=python$MAJOR_VERSION$MINOR_VERSION
+
+if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then
+    # No AGENT_TOOLSDIRECTORY on GitHub images
+    TOOLCACHE_ROOT=$RUNNER_TOOL_CACHE
+else
+    TOOLCACHE_ROOT=$AGENT_TOOLSDIRECTORY
+fi
+
+PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
+PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
+PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
+PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
+
+echo "Check if Python hostedtoolcache folder exist..."
+if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
+    echo "Creating Python hostedtoolcache folder..."
+    mkdir -p $PYTHON_TOOLCACHE_PATH
+else
+    # remove ALL other directories for same major.minor python versions
+    find $PYTHON_TOOLCACHE_PATH -name "${MAJOR_VERSION}.${MINOR_VERSION}.*"|while read python_version;do
+        python_version_arch="$python_version/$ARCH"
+        if [ -e "$python_version_arch" ];then
+            echo "Deleting Python $python_version_arch"
+            rm -rf "$python_version_arch"
+        fi
+    done
+fi
+
+echo "Install Python binaries from prebuilt package"
+sudo installer -pkg $PYTHON_PKG_NAME -target /
+
+echo "Create hostedtoolcach symlinks (Required for the backward compatibility)"
+echo "Create Python $PYTHON_FULL_VERSION folder"
+mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+
+ln -s "${PYTHON_FRAMEWORK_PATH}/bin" bin
+ln -s "${PYTHON_FRAMEWORK_PATH}/include" include
+ln -s "${PYTHON_FRAMEWORK_PATH}/share" share
+ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib
+
+echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
+ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
+
+cd bin/
+
+# This symlink already exists if Python version with the same major.minor version is installed,
+# since we do not remove the framework folder
+if [ ! -f $PYTHON_MAJOR_MINOR ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
+fi
+
+if [ ! -f python ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR python
+fi
+
+chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
+
+echo "Upgrading pip..."
+./python -m ensurepip
+./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location --root-user-action=ignore
+
+echo "Install OpenSSL certificates"
+sh -e "${PYTHON_APPLICATION_PATH}/Install Certificates.command"
+
+echo "Create complete file"
+touch $PYTHON_TOOLCACHE_VERSION_PATH/${ARCH}.complete

installers/nix-setup-template.sh

@@ -50,7 +50,7 @@ chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR pyth
 
 echo "Upgrading pip..."
 ./python -m ensurepip
-./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location
+./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location --root-user-action=ignore
 
 echo "Create complete file"
 touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete

installers/win-setup-template.ps1

@@ -101,7 +101,9 @@ if ($null -ne $InstalledVersions) {
         if (Test-Path -Path $InstalledVersion) {
             Write-Host "Deleting $InstalledVersion..."
             Remove-Item -Path $InstalledVersion -Recurse -Force
-            Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose
+            if (Test-Path -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete") {
+                Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose
+            }
         }
     }
 } else {
@@ -132,7 +134,7 @@ if ($MajorVersion -ne "2") {
 
 Write-Host "Install and upgrade Pip"
 $PythonExePath = Join-Path -Path $PythonArchPath -ChildPath "python.exe"
-cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip --no-warn-script-location"
+cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip --no-warn-script-location --root-user-action=ignore"
 
 Write-Host "Create complete file"
 New-Item -ItemType File -Path $PythonVersionPath -Name "$Architecture.complete" | Out-Null

tests/ManifestConfig.Tests.ps1

@@ -5,16 +5,16 @@ $Configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
 
 $stableTestCases = @(
     @{ ReleaseName = "python-3.8.3-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
-    @{ ReleaseName = "python-3.8.3-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
 ) | ForEach-Object { $_.Configuration = $Configuration; $_ }
 
 $unstableTestCases = @(
     @{ ReleaseName = "python-3.9.0-alpha.2-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
-    @{ ReleaseName = "python-3.9.0-rc.4-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-rc.4-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
 ) | ForEach-Object { $_.Configuration = $Configuration; $_ }

tests/python-tests.ps1

@@ -2,7 +2,9 @@ param (
     [semver] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
     $Version,
     [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
-    $Platform
+    $Platform,
+    [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    $Architecture
 )
 
 Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
@@ -49,7 +51,14 @@ Describe "Tests" {
         "python ./sources/simple-test.py" | Should -ReturnZeroExitCode
     }
 
-    if (($Version -ge "3.2.0") -and -not ([semver]"$($Version.Major).$($Version.Minor)" -eq [semver]"3.11" -and $Version.PreReleaseLabel)) {
+    # linux has no display name and no $DISPLAY environment variable - skip tk test
+    # if (-not (($Platform -match "ubuntu") -or ($Platform -match "linux"))) {
+    #     It "Check if tcl/tk has the same headed and library versions" {
+	#     "python ./sources/tcltk.py" | Should -ReturnZeroExitCode
+    #     }
+    # }
+
+    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($Architecture -ne "arm64"))) {
         It "Check if sqlite3 module is installed" {
             "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
         }
@@ -73,7 +82,7 @@ Describe "Tests" {
 
         It "Check if python configuration is correct" {
             $nativeVersion = Convert-Version -version $Version
-            "python ./sources/python-config-test.py $Version $nativeVersion" | Should -ReturnZeroExitCode
+            "python ./sources/python-config-test.py $Version $nativeVersion $Architecture" | Should -ReturnZeroExitCode
         }
 
         It "Check if shared libraries are linked correctly" {
@@ -86,7 +95,12 @@ Describe "Tests" {
         It "Validate Pyinstaller" {
             "pip install pyinstaller" | Should -ReturnZeroExitCode
             "pyinstaller --onefile ./sources/simple-test.py" | Should -ReturnZeroExitCode
-            "./dist/simple-test" | Should -ReturnZeroExitCode
+            $distPath = [IO.Path]::Combine($pwd, "dist", "simple-test")
+            "$distPath" | Should -ReturnZeroExitCode
         }
     }
+
+    It "Check urlopen with HTTPS works" {
+        "python ./sources/python-urlopen-https.py" | Should -ReturnZeroExitCode
+    }
 }

tests/sources/python-config-output.py

@@ -1,6 +1,4 @@
-import distutils.sysconfig
 import sysconfig
 
 from pprint import pprint
 pprint(sysconfig.get_config_vars())
-pprint(distutils.sysconfig.get_config_vars())

tests/sources/python-config-test.py

@@ -1,5 +1,3 @@
-import distutils.sysconfig
-from distutils.version import LooseVersion
 import sysconfig
 import sys
 import platform
@@ -9,6 +7,13 @@ import os
 os_type = platform.system()
 version = sys.argv[1]
 nativeVersion = sys.argv[2]
+architecture = sys.argv[3]
+
+versions=version.split(".")
+version_major=int(versions[0])
+version_minor=int(versions[1])
+
+pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (architecture == "arm64"))
 
 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
@@ -19,7 +24,11 @@ have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE")
 ### Define expected variables
 if os_type == 'Linux': expected_ld_library_extension = 'so'
 if os_type == 'Darwin': expected_ld_library_extension = 'dylib'
-expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version)
+
+if pkg_installer:
+    expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib'
+else:
+    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/x64/lib'
 
 # Check modules
 ### Validate libraries path
@@ -30,7 +39,7 @@ if lib_dir_path != expected_lib_dir_path:
 ### Validate shared libraries
 if is_shared:
     print('%s was built with shared extensions' % ld_library_name)
-    
+
     ### Validate libpython extension
     ld_library_extension = ld_library_name.split('.')[-1]
     if ld_library_extension != expected_ld_library_extension:
@@ -38,12 +47,13 @@ if is_shared:
         exit(1)
 else:
     print('%s was built without shared extensions' % ld_library_name)
-    exit(1)
+    if not pkg_installer:
+        exit(1)
 
 ### Validate macOS
 if os_type == 'Darwin':
     ### Validate openssl links
-    if LooseVersion(nativeVersion) < LooseVersion("3.7.0"):
+    if version_major == 3 and version_minor < 7:
         expected_ldflags = '-L/usr/local/opt/openssl@1.1/lib'
         ldflags = sysconfig.get_config_var('LDFLAGS')
 
@@ -53,18 +63,21 @@ if os_type == 'Darwin':
     else:
         expected_openssl_includes = '-I/usr/local/opt/openssl@1.1/include'
         expected_openssl_ldflags ='-L/usr/local/opt/openssl@1.1/lib'
-        
+
         openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES')
         openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS')
 
         if openssl_includes != expected_openssl_includes:
             print('Invalid openssl_includes: %s; Expected: %s' % (openssl_includes, expected_openssl_includes))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
         if openssl_ldflags != expected_openssl_ldflags:
             print('Invalid openssl_ldflags: %s; Expected: %s' % (openssl_ldflags, expected_openssl_ldflags))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
 
 ### Validate libreadline
-if not have_libreadline:
-    print('Missing libreadline')
-    exit(1)
+if sys.version_info < (3, 12):
+    if not have_libreadline:
+        print('Missing libreadline')
+        exit(1)

tests/sources/python-modules.py

@@ -7,7 +7,6 @@ from __future__ import print_function
 
 import importlib
 import sys
-import platform
 
 # The Python standard library as of Python 3.0
 standard_library = [
@@ -266,10 +265,14 @@ if sys.version_info >= (3, 10):
 if sys.version_info >= (3, 11):
     standard_library.remove('binhex')
 
-# Exclude tkinter and turtle for Python 3.11 alpha temporarily
-if sys.version_info >= (3, 11) and platform.system() == 'Linux' and '18.04' in platform.version():
-    standard_library.remove('tkinter')
-    standard_library.remove('turtle')
+# 'smtpd', 'asyncore' and 'asynchat' modules have been removed from Python 3.12
+# https://docs.python.org/dev/whatsnew/3.12.html
+if sys.version_info >= (3, 12):
+    standard_library.remove('distutils')
+    standard_library.remove('imp')
+    standard_library.remove('smtpd')
+    standard_library.remove('asyncore')
+    standard_library.remove('asynchat')
 
 # Remove tkinter and Easter eggs
 excluded_modules = [

tests/sources/python-urlopen-https.py

@@ -0,0 +1,10 @@
+import sys
+
+if sys.version_info[0] == 2:
+    from urllib2 import urlopen
+else:
+    from urllib.request import urlopen
+
+response = urlopen("https://raw.githubusercontent.com/actions/python-versions/c641695f6a07526c18f10e374e503e649fef9427/.gitignore")
+data = response.read()
+assert len(data) == 140, len(data)

tests/sources/tcltk.py

@@ -0,0 +1,10 @@
+import tkinter
+import _tkinter
+
+header = _tkinter.TK_VERSION
+lib = tkinter.Tk().getvar('tk_version')
+
+if lib != header:
+  print('header version=' + header)
+  print('lib version=' + lib)
+  exit(1)