Update versions-manifest (#260)

Co-authored-by: Service account <no-reply@microsoft.com>

.github/CODEOWNERS

@@ -1 +1 @@
-* @actions/virtual-environments-owners
+* @actions/setup-actions-team

.github/workflows/build-python-packages.yml

@@ -1,20 +1,21 @@
-name: Build python package
-
+name: Build Python package
+run-name: Generate Python ${{ inputs.VERSION || '3.11.0' }}
 on:
   workflow_dispatch:
     inputs:
       VERSION:
         description: 'Python version to build and upload'
-        default: '3.9.9'
+        default: '3.11.0'
         required: true
       PUBLISH_RELEASES:
         description: 'Whether to publish releases'
         required: true
-        default: 'false'
+        type: boolean
+        default: false
       PLATFORMS:
         description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
         required: true
-        default: 'ubuntu-18.04,ubuntu-20.04,ubuntu-22.04,macos-10.15,windows-2019_x64,windows-2019_x86'
+        default: 'ubuntu-20.04,ubuntu-22.04,macos-11_x64,macos-11_arm64,windows-2019_x64,windows-2019_x86'
   pull_request:
     paths-ignore:
     - 'versions-manifest.json'
@@ -24,7 +25,7 @@ on:
     - 'main'
 
 env:
-  VERSION: ${{ github.event.inputs.VERSION || '3.9.9' }}
+  VERSION: ${{ inputs.VERSION || '3.11.0' }}
 defaults:
   run:
     shell: pwsh
@@ -38,7 +39,7 @@ jobs:
       - name: Generate execution matrix
         id: generate-matrix
         run: |
-          $configurations = "${{ github.event.inputs.platforms || 'ubuntu-18.04,ubuntu-20.04,ubuntu-22.04,macos-10.15,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
+          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,macos-11,macos-11_arm64,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
           $matrix = @()
 
           foreach ($configuration in $configurations) {
@@ -56,7 +57,7 @@ jobs:
               'arch' = $arch
             }
           }
-          echo "::set-output name=matrix::$($matrix | ConvertTo-Json -Compress)"
+          echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT
 
   build_python:
     needs: generate_matrix
@@ -66,11 +67,11 @@ jobs:
         include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
     runs-on: ${{ matrix.os }}
     env:
-      ARTIFACT_NAME: python-${{ github.event.inputs.VERSION || '3.9.9' }}-${{ matrix.platform }}-${{ matrix.arch }}
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
     steps:
 
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           submodules: true
 
@@ -80,7 +81,7 @@ jobs:
                   -Platform ${{ matrix.platform }} -Architecture ${{ matrix.arch }}
 
       - name: Publish artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.ARTIFACT_NAME }}
           path: ${{ runner.temp }}/artifact
@@ -93,11 +94,11 @@ jobs:
         include: ${{ fromJson(needs.generate_matrix.outputs.matrix) }}
     runs-on: ${{ matrix.os }}
     env:
-      ARTIFACT_NAME: python-${{ github.event.inputs.VERSION || '3.9.9' }}-${{ matrix.platform }}-${{ matrix.arch }}
+      ARTIFACT_NAME: python-${{ inputs.VERSION || '3.11.0' }}-${{ matrix.platform }}-${{ matrix.arch }}
     steps:
 
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           submodules: true
 
@@ -105,7 +106,7 @@ jobs:
         run: ./helpers/clean-toolcache.ps1 -ToolName "Python"
 
       - name: Download artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           path: ${{ runner.temp }}
 
@@ -131,7 +132,7 @@ jobs:
         working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
 
       - name: Setup Python ${{ env.VERSION }}
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ env.VERSION }}
           architecture: ${{ matrix.arch }}
@@ -154,6 +155,7 @@ jobs:
             $pesterContainer = New-PesterContainer -Path './python-tests.ps1' -Data @{
                 Version="${{ env.VERSION }}";
                 Platform="${{ matrix.platform }}";
+                Architecture="${{ matrix.arch }}";
             }
             $Result = Invoke-Pester -Container $pesterContainer -PassThru
             if ($Result.FailedCount -gt 0) {
@@ -164,11 +166,11 @@ jobs:
 
   publish_release:
       name: Publish release
-      if: github.event_name == 'workflow_dispatch' && github.event.inputs.PUBLISH_RELEASES == 'true'
+      if: github.event_name == 'workflow_dispatch' && inputs.PUBLISH_RELEASES
       needs: test_python
       runs-on: ubuntu-latest
       steps:
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
 
       - name: Publish Release ${{ env.VERSION }}
         id: create_release
@@ -181,21 +183,34 @@ jobs:
           body: |
             Python ${{ env.VERSION }}
 
+      - name: Generate hash for packages
+        run: |
+          $childItems = Get-Childitem -Path '.'
+          $childItems | Foreach-Object {
+            $packageObj = Get-Childitem -Path $_.FullName | Select-Object -First 1
+            Write-Host "Package: $($packageObj.Name)"
+            $actualHash = (Get-FileHash -Path $packageObj.FullName -Algorithm sha256).Hash
+            $hashString = "$actualHash $($packageObj.Name)"
+            Write-Host "$hashString"
+            Add-Content -Path ./hashes.sha256 -Value "$hashString"
+          } 
+  
       - name: Upload release assets
-        uses: actions/github-script@v2
+        uses: actions/github-script@v6
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const fs = require('fs');
             for (let artifactDir of fs.readdirSync('.')) {
-              let artifactName = fs.readdirSync(`${artifactDir}`)[0];
+              let artifactName = fs.lstatSync(artifactDir).isDirectory() ? fs.readdirSync(`${artifactDir}`)[0] : artifactDir;
+  
               console.log(`Upload ${artifactName} asset`);
-              github.repos.uploadReleaseAsset({
+              github.rest.repos.uploadReleaseAsset({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 release_id: ${{ steps.create_release.outputs.id }},
                 name: artifactName,
-                data: fs.readFileSync(`./${artifactDir}/${artifactName}`)
+                data: fs.lstatSync(artifactDir).isDirectory() ? fs.readFileSync(`./${artifactDir}/${artifactName}`) : fs.readFileSync(`./${artifactName}`).toString()
               });
             }
 
@@ -205,11 +220,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Trigger "Create Pull Request" workflow
-      uses: actions/github-script@v3
+      uses: actions/github-script@v6
       with:
-        github-token: ${{ secrets.PERSONAL_TOKEN }}
+        github-token: ${{ secrets.GITHUB_TOKEN }}
         script: |
-          github.actions.createWorkflowDispatch({
+          github.rest.actions.createWorkflowDispatch({
             owner: context.repo.owner,
             repo: context.repo.repo,
             workflow_id: 'create-pr.yml',

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,16 @@
+name: CodeQL analysis
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 3 * * 0'
+
+jobs:
+  call-codeQL-analysis:
+    name: CodeQL analysis 
+    uses: actions/reusable-workflows/.github/workflows/codeql-analysis.yml@main
+    with:
+     languages: "['python']"

.github/workflows/create-pr.yml

@@ -2,32 +2,9 @@ name: Create Pull Request
 on:
   workflow_dispatch:
 
-defaults:
-  run:
-    shell: pwsh
-
 jobs:
-  create_pr:
-    name: Create Pull Request
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
+  create-pr:
+    uses: actions/versions-package-tools/.github/workflows/create-pr-to-update-manifest.yml@main
     with:
-        submodules: true
-
-    - name: Create versions-manifest.json
-      run: |
-        ./helpers/packages-generation/manifest-generator.ps1 -RepositoryFullName "$env:GITHUB_REPOSITORY" `
-                                                             -GitHubAccessToken "${{secrets.GITHUB_TOKEN}}" `
-                                                             -OutputFile "./versions-manifest.json" `
-                                                             -ConfigurationFile "./config/python-manifest-config.json"
-    - name: Create GitHub PR
-      run: |
-        $formattedDate = Get-Date -Format "MM/dd/yyyy"
-        ./helpers/github/create-pull-request.ps1 `
-                            -RepositoryFullName "$env:GITHUB_REPOSITORY" `
-                            -AccessToken "${{secrets.GITHUB_TOKEN}}" `
-                            -BranchName "update-versions-manifest-file" `
-                            -CommitMessage "Update versions-manifest" `
-                            -PullRequestTitle "[versions-manifest] Update for release from ${formattedDate}" `
-                            -PullRequestBody "Update versions-manifest.json for release from ${formattedDate}"
+      tool-name: "python"
+    secrets: inherit

.github/workflows/get-python-versions.yml

@@ -4,92 +4,10 @@ on:
     - cron: '0 3,15 * * *'
   workflow_dispatch:
 
-env:
-  TOOL_NAME: "Python"
-defaults:
-  run:
-    shell: pwsh
-
 jobs:
-  find_new_versions:
-    name: Find new versions
-    runs-on: ubuntu-latest
-    outputs:
-      versions_output: ${{ steps.Get_new_versions.outputs.TOOL_VERSIONS }}
-    steps:
-      - uses: actions/checkout@v2
+  get-new-python-versions:
+    uses: actions/versions-package-tools/.github/workflows/get-new-tool-versions.yml@main
     with:
-          submodules: true
-
-      - id: Get_new_versions
-        name: Get new versions
-        run: ./helpers/get-new-tool-versions/get-new-tool-versions.ps1 -ToolName ${{ env.TOOL_NAME }}
-
-  check_new_versions:
-    name: Check new versions
-    runs-on: ubuntu-latest
-    needs: find_new_versions
-    env:
-      TOOL_VERSIONS: ${{needs.find_new_versions.outputs.versions_output}}
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - name: Check Versions
-        if: env.TOOL_VERSIONS == ''
-        run: |
-          Write-Host "No new versions were found"
-          Import-Module "./helpers/github/github-api.psm1"
-          $gitHubApi = Get-GitHubApi -RepositoryFullName "$env:GITHUB_REPOSITORY" `
-                                     -AccessToken "${{ secrets.PERSONAL_TOKEN }}"
-          $gitHubApi.CancelWorkflow("$env:GITHUB_RUN_ID")
-          Start-Sleep -Seconds 60
-      - name: Send Slack notification
-        run: |
-          $pipelineUrl = "$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID"
-          $message = "The following versions of '${{ env.TOOL_NAME }}' are available to upload: ${{ env.TOOL_VERSIONS }}\nLink to the pipeline: $pipelineUrl"
-          ./helpers/get-new-tool-versions/send-slack-notification.ps1 -Url "${{ secrets.SLACK_CHANNEL_URL }}" `
-                                                                      -ToolName "${{ env.TOOL_NAME }}" `
-                                                                      -ImageUrl "https://avatars.githubusercontent.com/u/1525981?s=200&v=4" `
-                                                                      -Text "$message"
-  
-  trigger_builds:
-    name: Trigger builds
-    runs-on: ubuntu-latest
-    needs: [find_new_versions, check_new_versions]
-    env:
-      TOOL_VERSIONS: ${{needs.find_new_versions.outputs.versions_output}}
-    environment: Get Available Tools Versions - Publishing Approval
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - name: Trigger "Build python packages" workflow
-        run:
-          ./helpers/github/run-ci-builds.ps1 -RepositoryFullName "$env:GITHUB_REPOSITORY" `
-                                             -AccessToken "${{ secrets.PERSONAL_TOKEN }}" `
-                                             -WorkflowFileName "python-builder.yml" `
-                                             -WorkflowDispatchRef "main" `
-                                             -ToolVersions "${{ env.TOOL_VERSIONS }}" `
-                                             -PublishReleases "true"
-
-  check_build:
-    name: Check build for failures 
-    runs-on: ubuntu-latest
-    needs: [find_new_versions, check_new_versions, trigger_builds]
-    if: failure()
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - name: Send Slack notification if build fails
-        run: |
-          $pipelineUrl = "$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID"
-          $message = "The build of the '${{ env.TOOL_NAME }}' detection pipeline failed :progress-error:\nLink to the pipeline: $pipelineUrl"
-          ./helpers/get-new-tool-versions/send-slack-notification.ps1 -Url "${{ secrets.SLACK_CHANNEL_URL }}" `
-                                                                      -ToolName "${{ env.TOOL_NAME }}" `
-                                                                      -Text "$message" `
-                                                                      -ImageUrl "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+      tool-name: "Python"
+      image-url: "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+    secrets: inherit

.github/workflows/manifest-config-validation.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         submodules: true
 

.github/workflows/python-versions-runner.yml

@@ -21,11 +21,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       
       - name: Trigger python workflow
         run: |
           $versions = ${{ github.event.inputs.versions }}
           ./builders/python-versions-runner.ps1 -Versions $versions.Split(",") -PublishRelease ${{ github.event.inputs.publish-releases }}
         env:
-          PERSONAL_TOKEN: ${{ secrets.PERSONAL_TOKEN }}
+          TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/releases-validation.yml

@@ -14,11 +14,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-18.04, ubuntu-20.04]
-        python: [3.5.4, 3.6.7, 3.7.5, 3.8.1]
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04]
+        python: [3.9.12, 3.10.8, 3.11.8]
     steps:
     - name: setup-python ${{ matrix.python }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python }}
 

.github/workflows/test-python-version.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-18.04, ubuntu-20.04] 
+        os: [macos-latest, windows-latest, ubuntu-20.04, ubuntu-22.04] 
     steps:
     - name: Setup Python ${{ github.event.inputs.version }}
       uses: actions/setup-python@main

.github/workflows/validate-manifest.yml

@@ -1,48 +1,21 @@
 name: Validate manifest
-
 on:
+# The GITHUB_TOKEN secret is used to create a PR
+# The pull_request event will not be triggered by it
+# That's one of the reasons we need the schedule to validate the versions-manifest.json file
   schedule:
     - cron: '0 8,20 * * *'
-
+  workflow_dispatch:
   pull_request:
     branches:
       - main
     paths:
       - 'versions-manifest.json'
 
-env:
-  TOOL_NAME: "Python"
-
-defaults:
-  run:
-    shell: pwsh
-
 jobs:
-  validation:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
+  manifest:
+    uses: actions/versions-package-tools/.github/workflows/validate-manifest.yml@main
     with:
-          submodules: true
-
-      - name: Validate python-versions manifest
-        run: .\helpers\packages-generation\manifest-validator.ps1 -ManifestPath '.\versions-manifest.json'
-
-  check_build:
-    name: Check validation for failures 
-    runs-on: ubuntu-latest
-    needs: [validation]
-    if: failure()
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - name: Send Slack notification if validation fails
-        run: |
-          $pipelineUrl = "$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID"
-          $message = "The validation of python-versions manifest failed. \nLink to the pipeline: $pipelineUrl"
-          .\helpers\get-new-tool-versions\send-slack-notification.ps1 -Url "${{ secrets.SLACK_CHANNEL_URL }}" `
-                                                                      -ToolName "${{ env.TOOL_NAME }}" `
-                                                                      -Text "$message" `
-                                                                      -ImageUrl "https://www.python.org/static/community_logos/python-powered-h-100x130.png"
+      tool-name: "Python"
+      image-url: "https://avatars.githubusercontent.com/u/1525981?s=200&v=4"
+    secrets: inherit

README.md

@@ -1,16 +1,27 @@
 # Python for Actions
-This repository contains the code and scripts that we use to build Python packages used in [virtual-environments](https://github.com/actions/virtual-environments) and accessible through the [setup-python](https://github.com/actions/setup-python) Action.  
+This repository contains the code and scripts that we use to build Python packages used in [runner-images](https://github.com/actions/runner-images) and accessible through the [setup-python](https://github.com/actions/setup-python) Action.
 File [versions-manifest.json](./versions-manifest.json) contains the list of available and released versions.
 
-> Caution: this is prepared for and only permitted for use by actions `virtual-environments` and `setup-python` action.
+> Caution: this is prepared for and only permitted for use by actions `runner-images` and `setup-python` action.
 
-**Status**: Currently under development and in use for beta and preview actions.  This repo is undergoing rapid changes.
+Some versions are pre-installed on [runner-images](https://github.com/actions/runner-images) images.
+More versions are available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
 
-Some versions are pre-installed on [virtual-environments](https://github.com/actions/virtual-environments) images.  
-More versions will (soon!) be available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
+## Building installation packages
+**Ubuntu:** The official Python sources are downloaded from [python.org](https://www.python.org/ftp/python/), built using the `make` tool, and archived along with the installation script for further distribution and installation. We build each Python version against all [versions of Ubuntu](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) that are available at the time the Python version is released.
 
-## Adding new versions
-We are trying to build and release new versions of Python as soon as they are released. Please open an issue in [actions/virtual-environments](https://github.com/actions/virtual-environments) if any versions are missing.
+**macOS:** 
+- For **Python < 3.11**, the official Python sources are downloaded from [python.org](https://www.python.org/ftp/python/), built using the `make` tool, and archived along with the installation script for further distribution and installation.  For backward compatibility, we build each Python version against the oldest [version of macOS](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) that is available at the time the Python version is released.
+- For **Python >= 3.11**, the official macOS `universal2` Python binaries are simply downloaded from [python.org](https://www.python.org/ftp/python/) and archived along with the installation script for further distribution and installation.
+
+**Windows:** The official Python executables are simply downloaded from [python.org](https://www.python.org/ftp/python/) and archived along with the installation script for further distribution and installation.
+
+## Support policy
+We aim to make new versions of Python available as soon as they are released. Both stable and prerelease versions are considered for building and releasing. Please open an issue in [actions/setup-python](https://github.com/actions/setup-python) if a version you're looking for isn't available.
+
+When a new version of an operating system is released and made available for use with [GitHub hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources), we will provide the **latest existing patch versions of Python for all major versions that have not reached [end-of-life](https://devguide.python.org/versions/)**.  Any subsequent Python versions will be made available for the new OS as well.
+
+Beginning **approximately six months prior** to the removal of a Python version from the [versions-manifest.json](https://github.com/actions/python-versions/blob/main/versions-manifest.json) file, a pinned issue will be created in the [setup-python](https://github.com/actions/setup-python) repository. This pinned issue will provide important details about the upcoming end of support, including the specific date, as well as any other notes, relevant updates or alternatives. We encourage users to regularly check pinned issues for updates on tool versions they are using for maximum transparency, security, performance and overall compatibility with their projects.
 
 ## Contribution
 Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure

builders/invoke-workflow.psm1

@@ -12,10 +12,10 @@ function Invoke-Workflow {
         }
     } | ConvertTo-Json
     $headers = @{
-        Authorization="Bearer $env:PERSONAL_TOKEN"
+        Authorization="Bearer $env:TOKEN"
     }
     $actionsRepoUri = "$env:GITHUB_API_URL/repos/$env:GITHUB_REPOSITORY/actions"
-    Invoke-RestMethod -uri "$actionsRepoUri/workflows/python-builder.yml/dispatches" -method POST -headers $headers -body $payload
+    Invoke-RestMethod -uri "$actionsRepoUri/workflows/build-python-packages.yml/dispatches" -method POST -headers $headers -body $payload
 
     $result = [PSCustomObject]@{
         Version = $Version

builders/macos-python-builder.psm1

@@ -31,6 +31,14 @@ class macOSPythonBuilder : NixPythonBuilder {
         .SYNOPSIS
         Prepare system environment by installing dependencies and required packages.
         #>
+        
+        if ($this.Version -eq "3.7.17") {
+            # We have preinstalled ncurses and readLine on the hoster runners. But we need to install bzip2 for 
+            # setting up an environemnt 
+            # If we get any issues realted to ncurses or readline we can try to run this command
+            # brew install ncurses readline
+            Execute-Command -Command "brew install bzip2"
+        }
     }
 
     [void] Configure() {
@@ -64,9 +72,22 @@ class macOSPythonBuilder : NixPythonBuilder {
             $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/zlib/include"
         } else {
             $configureString += " --with-openssl=/usr/local/opt/openssl@1.1"
+
+            # For Python 3.7.2 and 3.7.3 we need to provide PATH for zlib to pack it properly. Otherwise the build will fail
+            # with the error: zipimport.ZipImportError: can't decompress data; zlib not available
+            if ($this.Version -eq "3.7.2" -or $this.Version -eq "3.7.3" -or $this.Version -eq "3.7.17") {
+                $env:LDFLAGS = "-L/usr/local/opt/zlib/lib"
+                $env:CFLAGS = "-I/usr/local/opt/zlib/include"
+            }
+
             if ($this.Version -gt "3.7.12") {
                 $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'"
 	        }
+
+            if ($this.Version -eq "3.7.17") {
+                $env:LDFLAGS += " -L$(brew --prefix bzip2)/lib -L$(brew --prefix readline)/lib -L$(brew --prefix ncurses)/lib"
+                $env:CFLAGS += " -I$(brew --prefix bzip2)/include -I$(brew --prefix readline)/include -I$(brew --prefix ncurses)/include"
+            }
         }
 
         ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
@@ -83,4 +104,93 @@ class macOSPythonBuilder : NixPythonBuilder {
 
         Execute-Command -Command $configureString
     }
+
+    [string] GetPkgName() {
+        <#
+        .SYNOPSIS
+        Return Python installation Package.
+        #>
+
+        $nativeVersion = Convert-Version -version $this.Version
+        $architecture = "-macos11"
+        $extension = ".pkg"
+
+        $pkg = "python-${nativeVersion}${architecture}${extension}"
+
+        return $pkg
+    }
+
+    [uri] GetPkgUri() {
+        <#
+        .SYNOPSIS
+        Get base Python URI and return complete URI for Python installation package.
+        #>
+
+        $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $pkg = $this.GetPkgName()
+
+        $uri = "${base}/${versionName}/${pkg}"
+
+        return $uri
+    }
+
+    [string] DownloadPkg() {
+        <#
+        .SYNOPSIS
+        Download Python installation executable into artifact location.
+        #>
+
+        $pkgUri = $this.GetPkgUri()
+
+        Write-Host "Sources URI: $pkgUri"
+        $pkgLocation = Download-File -Uri $pkgUri -OutputFolder $this.WorkFolderLocation
+        Write-Debug "Done; Package location: $pkgLocation"
+
+        New-Item -Path $this.WorkFolderLocation -Name "build_output.txt"  -ItemType File
+        return $pkgLocation
+    }
+
+    [void] CreateInstallationScriptPkg() {
+        <#
+        .SYNOPSIS
+        Create Python artifact installation script based on specified template.
+        #>
+
+        $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath "macos-pkg-setup-template.sh"
+        $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
+        $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName  -ItemType File
+
+        $variablesToReplace = @{
+            "{{__VERSION_FULL__}}" = $this.Version;
+            "{{__PKG_NAME__}}" = $this.GetPkgName();
+            "{{__ARCH__}}" = $this.Architecture;
+        }
+
+        $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
+        $installationTemplateContent | Out-File -FilePath $installationScriptLocation
+        Write-Debug "Done; Installation script location: $installationScriptLocation)"
+    }
+
+    [void] Build() {
+        <#
+        .SYNOPSIS
+        Generates Python artifact from downloaded Python installation executable.
+        #>
+
+        $PkgVersion = [semver]"3.11.0-beta.1"
+
+        if (($this.Version -ge $PkgVersion) -or ($this.Architecture -eq "arm64")) {
+            Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..."
+            $this.DownloadPkg()
+
+            Write-Host "Create installation script..."
+            $this.CreateInstallationScriptPkg()
+        } else {
+            ([NixPythonBuilder]$this).Build()
+        }
+
+        Write-Host "Archive artifact"
+        $this.ArchiveArtifact()
+    }
 }

builders/ubuntu-python-builder.psm1

@@ -72,6 +72,7 @@ class UbuntuPythonBuilder : NixPythonBuilder {
             $tkinterInstallString = "sudo apt install -y python-tk tk-dev"
         }
 
+        Execute-Command -Command "sudo apt-get update"
         Execute-Command -Command $tkinterInstallString
 
         ### Install dependent packages

config/python-manifest-config.json

@@ -1,5 +1,5 @@
 {
-    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?(x\\d+)",
+    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+)",
     "groups": {
         "arch": 4,
         "platform": 2,

installers/macos-pkg-setup-template.sh

@@ -0,0 +1,79 @@
+set -e
+
+PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
+PYTHON_PKG_NAME="{{__PKG_NAME__}}"
+ARCH="{{__ARCH__}}"
+MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
+MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
+
+PYTHON_MAJOR=python$MAJOR_VERSION
+PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION
+PYTHON_MAJOR_MINOR=python$MAJOR_VERSION$MINOR_VERSION
+
+if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then
+    # No AGENT_TOOLSDIRECTORY on GitHub images
+    TOOLCACHE_ROOT=$RUNNER_TOOL_CACHE
+else
+    TOOLCACHE_ROOT=$AGENT_TOOLSDIRECTORY
+fi
+
+PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
+PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
+PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
+PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
+
+echo "Check if Python hostedtoolcache folder exist..."
+if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
+    echo "Creating Python hostedtoolcache folder..."
+    mkdir -p $PYTHON_TOOLCACHE_PATH
+else
+    # remove ALL other directories for same major.minor python versions
+    find $PYTHON_TOOLCACHE_PATH -name "${MAJOR_VERSION}.${MINOR_VERSION}.*"|while read python_version;do
+        python_version_arch="$python_version/$ARCH"
+        if [ -e "$python_version_arch" ];then
+            echo "Deleting Python $python_version_arch"
+            rm -rf "$python_version_arch"
+        fi
+    done
+fi
+
+echo "Install Python binaries from prebuilt package"
+sudo installer -pkg $PYTHON_PKG_NAME -target /
+
+echo "Create hostedtoolcach symlinks (Required for the backward compatibility)"
+echo "Create Python $PYTHON_FULL_VERSION folder"
+mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+
+ln -s "${PYTHON_FRAMEWORK_PATH}/bin" bin
+ln -s "${PYTHON_FRAMEWORK_PATH}/include" include
+ln -s "${PYTHON_FRAMEWORK_PATH}/share" share
+ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib
+
+echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
+ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
+
+cd bin/
+
+# This symlink already exists if Python version with the same major.minor version is installed,
+# since we do not remove the framework folder
+if [ ! -f $PYTHON_MAJOR_MINOR ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
+fi
+
+if [ ! -f python ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR python
+fi
+
+chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
+
+echo "Upgrading pip..."
+./python -m ensurepip
+./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location --root-user-action=ignore
+
+echo "Install OpenSSL certificates"
+sh -e "${PYTHON_APPLICATION_PATH}/Install Certificates.command"
+
+echo "Create complete file"
+touch $PYTHON_TOOLCACHE_VERSION_PATH/${ARCH}.complete

installers/nix-setup-template.sh

@@ -50,7 +50,7 @@ chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR pyth
 
 echo "Upgrading pip..."
 ./python -m ensurepip
-./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location
+./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location --root-user-action=ignore
 
 echo "Create complete file"
 touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete

installers/win-setup-template.ps1

@@ -101,9 +101,11 @@ if ($null -ne $InstalledVersions) {
         if (Test-Path -Path $InstalledVersion) {
             Write-Host "Deleting $InstalledVersion..."
             Remove-Item -Path $InstalledVersion -Recurse -Force
+            if (Test-Path -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete") {
                 Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose
             }
         }
+    }
 } else {
     Write-Host "No Python$MajorVersion.$MinorVersion.* found"
 }
@@ -132,7 +134,7 @@ if ($MajorVersion -ne "2") {
 
 Write-Host "Install and upgrade Pip"
 $PythonExePath = Join-Path -Path $PythonArchPath -ChildPath "python.exe"
-cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip --no-warn-script-location"
+cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip --no-warn-script-location --root-user-action=ignore"
 
 Write-Host "Create complete file"
 New-Item -ItemType File -Path $PythonVersionPath -Name "$Architecture.complete" | Out-Null

tests/ManifestConfig.Tests.ps1

@@ -5,16 +5,16 @@ $Configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
 
 $stableTestCases = @(
     @{ ReleaseName = "python-3.8.3-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
-    @{ ReleaseName = "python-3.8.3-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.8.3-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
     @{ ReleaseName = "python-3.8.3-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
 ) | ForEach-Object { $_.Configuration = $Configuration; $_ }
 
 $unstableTestCases = @(
     @{ ReleaseName = "python-3.9.0-alpha.2-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} },
-    @{ ReleaseName = "python-3.9.0-rc.4-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} },
+    @{ ReleaseName = "python-3.9.0-rc.4-linux-22.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "22.04"; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} },
     @{ ReleaseName = "python-3.9.0-beta.2-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} }
 ) | ForEach-Object { $_.Configuration = $Configuration; $_ }

tests/python-tests.ps1

@@ -2,7 +2,9 @@ param (
     [semver] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
     $Version,
     [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
-    $Platform
+    $Platform,
+    [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    $Architecture
 )
 
 Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
@@ -56,7 +58,7 @@ Describe "Tests" {
     #     }
     # }
 
-    if (($Version -ge "3.2.0") -and -not ([semver]"$($Version.Major).$($Version.Minor)" -eq [semver]"3.11" -and $Version.PreReleaseLabel)) {
+    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($Architecture -ne "arm64"))) {
         It "Check if sqlite3 module is installed" {
             "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
         }
@@ -80,7 +82,7 @@ Describe "Tests" {
 
         It "Check if python configuration is correct" {
             $nativeVersion = Convert-Version -version $Version
-            "python ./sources/python-config-test.py $Version $nativeVersion" | Should -ReturnZeroExitCode
+            "python ./sources/python-config-test.py $Version $nativeVersion $Architecture" | Should -ReturnZeroExitCode
         }
 
         It "Check if shared libraries are linked correctly" {
@@ -93,7 +95,12 @@ Describe "Tests" {
         It "Validate Pyinstaller" {
             "pip install pyinstaller" | Should -ReturnZeroExitCode
             "pyinstaller --onefile ./sources/simple-test.py" | Should -ReturnZeroExitCode
-            "./dist/simple-test" | Should -ReturnZeroExitCode
+            $distPath = [IO.Path]::Combine($pwd, "dist", "simple-test")
+            "$distPath" | Should -ReturnZeroExitCode
         }
     }
+
+    It "Check urlopen with HTTPS works" {
+        "python ./sources/python-urlopen-https.py" | Should -ReturnZeroExitCode
+    }
 }

tests/sources/python-config-output.py

@@ -1,6 +1,4 @@
-import distutils.sysconfig
 import sysconfig
 
 from pprint import pprint
 pprint(sysconfig.get_config_vars())
-pprint(distutils.sysconfig.get_config_vars())

tests/sources/python-config-test.py

@@ -1,5 +1,3 @@
-import distutils.sysconfig
-from distutils.version import LooseVersion
 import sysconfig
 import sys
 import platform
@@ -9,6 +7,13 @@ import os
 os_type = platform.system()
 version = sys.argv[1]
 nativeVersion = sys.argv[2]
+architecture = sys.argv[3]
+
+versions=version.split(".")
+version_major=int(versions[0])
+version_minor=int(versions[1])
+
+pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (architecture == "arm64"))
 
 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
@@ -19,7 +24,11 @@ have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE")
 ### Define expected variables
 if os_type == 'Linux': expected_ld_library_extension = 'so'
 if os_type == 'Darwin': expected_ld_library_extension = 'dylib'
-expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version)
+
+if pkg_installer:
+    expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib'
+else:
+    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/x64/lib'
 
 # Check modules
 ### Validate libraries path
@@ -38,12 +47,13 @@ if is_shared:
         exit(1)
 else:
     print('%s was built without shared extensions' % ld_library_name)
+    if not pkg_installer:
         exit(1)
 
 ### Validate macOS
 if os_type == 'Darwin':
     ### Validate openssl links
-    if LooseVersion(nativeVersion) < LooseVersion("3.7.0"):
+    if version_major == 3 and version_minor < 7:
         expected_ldflags = '-L/usr/local/opt/openssl@1.1/lib'
         ldflags = sysconfig.get_config_var('LDFLAGS')
 
@@ -59,12 +69,15 @@ if os_type == 'Darwin':
 
         if openssl_includes != expected_openssl_includes:
             print('Invalid openssl_includes: %s; Expected: %s' % (openssl_includes, expected_openssl_includes))
+            if not pkg_installer:
                 exit(1)
         if openssl_ldflags != expected_openssl_ldflags:
             print('Invalid openssl_ldflags: %s; Expected: %s' % (openssl_ldflags, expected_openssl_ldflags))
+            if not pkg_installer:
                 exit(1)
 
 ### Validate libreadline
+if sys.version_info < (3, 12):
     if not have_libreadline:
         print('Missing libreadline')
         exit(1)

tests/sources/python-modules.py

@@ -7,7 +7,6 @@ from __future__ import print_function
 
 import importlib
 import sys
-import platform
 
 # The Python standard library as of Python 3.0
 standard_library = [
@@ -266,10 +265,35 @@ if sys.version_info >= (3, 10):
 if sys.version_info >= (3, 11):
     standard_library.remove('binhex')
 
-# Exclude tkinter and turtle for Python 3.11 alpha temporarily
-if sys.version_info >= (3, 11) and platform.system() == 'Linux' and '18.04' in platform.version():
-    standard_library.remove('tkinter')
-    standard_library.remove('turtle')
+# 'smtpd', 'asyncore' and 'asynchat' modules have been removed from Python 3.12
+# https://docs.python.org/dev/whatsnew/3.12.html
+if sys.version_info >= (3, 12):
+    standard_library.remove('distutils')
+    standard_library.remove('imp')
+    standard_library.remove('smtpd')
+    standard_library.remove('asyncore')
+    standard_library.remove('asynchat')
+
+# 'aifc', 'cgi', 'cgitb', 'chunk', 'crypt', 'imghdr', 'lib2to3', 'mailcap', 'nntplib',
+# 'pipes', 'sndhdr', 'sunau', 'telnetlib', 'uu' and 'xdrlib' modules have been removed
+# from Python 3.13
+# https://docs.python.org/dev/whatsnew/3.13.html
+if sys.version_info >= (3, 13):
+    standard_library.remove('aifc')
+    standard_library.remove('cgi')
+    standard_library.remove('cgitb')
+    standard_library.remove('chunk')
+    standard_library.remove('crypt')
+    standard_library.remove('imghdr')
+    standard_library.remove('lib2to3')
+    standard_library.remove('mailcap')
+    standard_library.remove('nntplib')
+    standard_library.remove('pipes')
+    standard_library.remove('sndhdr')
+    standard_library.remove('sunau')
+    standard_library.remove('telnetlib')
+    standard_library.remove('uu')
+    standard_library.remove('xdrlib')
 
 # Remove tkinter and Easter eggs
 excluded_modules = [

tests/sources/python-urlopen-https.py

@@ -0,0 +1,10 @@
+import sys
+
+if sys.version_info[0] == 2:
+    from urllib2 import urlopen
+else:
+    from urllib.request import urlopen
+
+response = urlopen("https://raw.githubusercontent.com/actions/python-versions/c641695f6a07526c18f10e374e503e649fef9427/.gitignore")
+data = response.read()
+assert len(data) == 140, len(data)