Merge branch 'main' of https://github.com/actions/virtual-environments into spotlight

.github/ISSUE_TEMPLATE/bug-report.md

@@ -17,10 +17,11 @@ A clear and concise description of what the bug is, and why you consider it to b
 <!-- Choose "Question", "Bug", or "Feature" -->
 
 **Virtual environments affected**  
+- [ ] Ubuntu 16.04
+- [ ] Ubuntu 18.04
+- [ ] Ubuntu 20.04
 - [ ] macOS 10.15
-- [ ] Ubuntu 16.04 LTS
+- [ ] macOS 11.0
-- [ ] Ubuntu 18.04 LTS
-- [ ] Ubuntu 20.04 LTS
 - [ ] Windows Server 2016 R2
 - [ ] Windows Server 2019
 

.github/ISSUE_TEMPLATE/tool-request.md

@@ -25,10 +25,11 @@ assignees: ''
 <!-- Choose "Question", "Bug", or "Feature" -->
 
 **Virtual environments affected**
+- [ ] Ubuntu 16.04
+- [ ] Ubuntu 18.04
+- [ ] Ubuntu 20.04
 - [ ] macOS 10.15
-- [ ] Ubuntu 16.04 LTS
+- [ ] macOS 11.0
-- [ ] Ubuntu 18.04 LTS
-- [ ] Ubuntu 20.04 LTS
 - [ ] Windows Server 2016 R2
 - [ ] Windows Server 2019
 

.github/workflows/linter.yml

@@ -1,10 +1,10 @@
 # CI Validation
 
-name: CI
+name: Linter
 
 on:
   pull_request:
-    branches: [$default-branch]
+    branches: [ main ]
 
 jobs:
   build:
@@ -23,3 +23,7 @@ jobs:
           VALIDATE_JSON: true
           VALIDATE_MD: true
           DEFAULT_BRANCH: ${{ github.base_ref }}
+
+      - name: Checking shebang lines in MacOS and Ubuntu releases.
+        run: ./images.CI/shebang-linter.ps1
+        shell: pwsh

README.md

@@ -9,13 +9,18 @@ For general questions about using the virtual environments or writing your Actio
 | Ubuntu 20.04 | `ubuntu-20.04` | [ubuntu-20.04] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu20&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu20&redirect=1)
 | Ubuntu 18.04 | `ubuntu-latest` or `ubuntu-18.04` | [ubuntu-18.04] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu18&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu18&redirect=1)
 | Ubuntu 16.04 | `ubuntu-16.04` | [ubuntu-16.04] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu16&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu16&redirect=1) |
-| macOS 10.15 | `macos-latest` or `macos-10.15` | [macOS-10.15] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=MacOS&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=MacOS&redirect=1)
+| macOS 11.0 | `macos-11.0` | [macOS-11.0] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=macos-11.0&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=macos-11.0&redirect=1)
+| macOS 10.15 | `macos-latest` or `macos-10.15` | [macOS-10.15] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=macos-10.15&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=macos-10.15&redirect=1)
 | Windows Server 2019 | `windows-latest` or `windows-2019` | [windows-2019] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2019&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2019&redirect=1)
 | Windows Server 2016 | `windows-2016` | [windows-2016] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2016&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2016&redirect=1)
 ```
 The Ubuntu 20.04 virtual environment is currently provided as a preview only.
 The "ubuntu-latest" YAML workflow label still uses the Ubuntu 18.04 virtual environment.
 ```
+```
+The MacOS 11.0 virtual environment is currently provided as a preview only.
+The "macos-latest" YAML workflow label still uses the MacOS 10.15 virtual environment.
+```
 
 ***Looking for other Linux distributions?*** We do not plan to offer other Linux distributions. We recommend using Docker if you'd like to build using other distributions with the hosted virtual environments. Alternatively, you can leverage [self-hosted runners] and fully customize your environment to your needs.
 
@@ -56,4 +61,5 @@ Low Impact changes will be pinned in this repository and marked with the [Announ
 [Windows-2019]: https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md
 [windows-2016]: https://github.com/actions/virtual-environments/blob/main/images/win/Windows2016-Readme.md
 [macOS-10.15]: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md
+[macOS-11.0]: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11.0-Readme.md
 [self-hosted runners]: https://help.github.com/en/actions/hosting-your-own-runners

images.CI/macos/azure-pipelines/image-generation.yml

@@ -52,7 +52,8 @@ jobs:
     inputs:
       targetType: 'filePath'
       filePath: ./images.CI/macos/select-datastore.ps1
-      arguments: -VIServer "$(vcenter-server-v2)" `
+      arguments: -VMName "$(VirtualMachineName)" `
+                 -VIServer "$(vcenter-server-v2)" `
                  -VIUserName "$(vcenter-username-v2)" `
                  -VIPassword "$(vcenter-password-v2)"
 
@@ -121,8 +122,8 @@ jobs:
     condition: always()
 
   - task: PowerShell@2
-    displayName: 'Move vm to cold storage'
+    displayName: 'Move vm to cold storage and clear datastore tag'
-    condition: succeededOrFailed()
+    condition: always()
     inputs:
       targetType: 'filePath'
       filePath: ./images.CI/macos/move-vm.ps1

images.CI/macos/move-vm.ps1

@@ -48,6 +48,13 @@ Import-Module $PSScriptRoot\helpers.psm1 -DisableNameChecking
 # Connection to a vCenter Server system
 Connect-VCServer
 
+# Clear previously assigned tag with VM Name
+try {
+    Remove-Tag $VMName -Confirm:$false
+} catch {
+    Write-Host "Tag with $VMName doesn't exist"
+}
+
 $vm = Get-VM $VMName
 
 if ($env:AGENT_JOBSTATUS -eq 'Failed') {

images.CI/macos/select-datastore.ps1

@@ -20,6 +20,10 @@ vCenter password (Example "12345678")
 
 [CmdletBinding()]
 param(
+    [Parameter(Mandatory)]
+    [ValidateNotNullOrEmpty()]
+    [string]$VMName,
+
     [Parameter(Mandatory)]
     [ValidateNotNullOrEmpty()]
     [string]$VIServer,
@@ -30,35 +34,72 @@ param(
 
     [Parameter(Mandatory)]
     [ValidateNotNullOrEmpty()]
-    [string]$VIPassword
+    [string]$VIPassword,
+
+    [string]$TagCategory = "Busy"
 )
 
 # Import helpers module
 Import-Module $PSScriptRoot\helpers.psm1 -DisableNameChecking
 
+function Select-DataStore {
+    param (
+        [string]$VMName,
+        [string]$TagCategory,
+        [string]$TemplateDatastore = "ds-local-Datastore-*",
+        [int]$ThresholdInGb = 400,
+        [int]$VMCount = 2,
+        [int]$Retries = 5
+    )
+
+    # 1. Name starts with ds-local-Datastore
+    # 2. FreespaceGB > 400 Gb
+    # 3. VM count on a datastore < 2
+
+    Write-Host "Start Datastore selection process..."
+    $allDatastores = Get-Datastore -Name $templateDatastore | Where-Object { $_.State -eq "Available" }
+    $buildDatastore = $allDatastores | Where-Object { $_.FreeSpaceGB -ge $thresholdInGb } | Where-Object {
+        $vmOnDatastore = @((Get-ChildItem -Path $_.DatastoreBrowserPath).Name -notmatch "^\.").Count
+        $vmOnDatastore -lt $vmCount
+    } | Select-Object -ExpandProperty Name -First 1
+
+    $tag = Get-Tag -Category $TagCategory -Name $VMName -ErrorAction Ignore
+    if (-not $tag)
+    {
+        $tag = New-Tag -Name $VMName -Category $TagCategory
+    }
+
+    New-TagAssignment -Tag $tag -Entity $buildDatastore | Out-Null
+
+    # Wait for 60 seconds to check if any other tags are assigned to the same datastore
+    Start-Sleep -Seconds 60
+    # Take only first 2 tags, all the others will go to the next round
+    $tagAssignments = (Get-TagAssignment -Entity $buildDatastore).Tag.Name | Select-Object -First 2
+    $isAllow = $tagAssignments -contains $VMName
+
+    if ($isAllow)
+    {
+        Write-Host "Datastore selected successfully"
+        Write-Host "##vso[task.setvariable variable=buildDatastore;issecret=true]$buildDatastore"
+        return
+    }
+
+    # Remove the tag if datastore wasn't selected
+    Remove-Tag $tag -Confirm:$false
+
+    $retries--
+    if ($retries -le 0)
+    {
+        Write-Host "##vso[task.LogIssue type=error;]No datastores found for the condition"
+        exit 1
+    }
+
+    Write-Host "Datastore select failed, $retries left"
+    Select-DataStore -VMName $VMName -TagCategory $TagCategory -Retries $retries
+}
+
 # Connection to a vCenter Server system
 Connect-VCServer
 
 # Get a target datastore for current deployment
-# 1. Name starts with ds-local-Datastore
+Select-DataStore -VMName $VMName -TagCategory $TagCategory
-# 2. FreespaceGB > 400 Gb
-# 3. VM count on a datastore < 2
-$templateDatastore = "ds-local-Datastore-*"
-$thresholdInGb = 400
-$vmCount = 2
-$allDatastores = Get-Datastore -Name $templateDatastore | Where-Object { $_.State -eq "Available" }
-$buildDatastore = $allDatastores | Where-Object { $_.FreeSpaceGB -ge $thresholdInGb } | Where-Object {
-            $vmOnDatastore = @((Get-ChildItem -Path $_.DatastoreBrowserPath).Name -notmatch "^\.").Count
-            $vmOnDatastore -lt $vmCount
-        } | Select-Object -ExpandProperty Name -First 1
-
-if ($buildDatastore)
-{
-    Write-Host "Datastore selected successfully"
-    Write-Host "##vso[task.setvariable variable=buildDatastore;issecret=true]$buildDatastore"
-}
-else
-{
-    Write-Host "##vso[task.LogIssue type=error;]No datastores found for the condition"
-    exit 1
-}

images.CI/shebang-linter.ps1

@@ -0,0 +1,43 @@
+$ErrorActionPreference = "Stop"
+
+function Validate-Scripts {
+    Param (
+        [Parameter(Mandatory=$true)]
+        [string[]]$Path,
+        [Parameter(Mandatory=$true)]
+        [string]$ExpectedShebang
+    )
+    $ScriptWithoutShebangLine = @()
+    Get-ChildItem $path -Recurse -File -Filter "*.sh" | ForEach-Object {
+        $relativePath = Resolve-Path $_.FullName -Relative
+        $shebangLine = Get-Content -Path $_.FullName | Select-Object -First 1
+        if ($shebangLine -eq $ExpectedShebang) {
+            Write-Host "[+] '$relativePath'"
+        }
+        else {
+            Write-Host "[-] '$relativePath'"
+            $ScriptWithoutShebangLine += $relativePath
+        }
+    }
+    return $ScriptWithoutShebangLine
+}
+
+$PathUbuntu = "./images/linux/scripts"
+$PathMacOS = "./images/macos/provision"
+$PatternUbuntu = "#!/bin/bash -e"
+$PatternMacOS = "#!/bin/bash -e -o pipefail"
+$ScriptsWithBrokenShebang = @()
+$ScriptsWithBrokenShebang += Validate-Scripts -Path $PathUbuntu -ExpectedShebang $PatternUbuntu
+$ScriptsWithBrokenShebang += Validate-Scripts -Path $PathMacOS -ExpectedShebang $PatternMacOS
+if ($ScriptsWithBrokenShebang.Length -gt 0) {
+    Write-Host "`n`n`n##[error] The following scripts have incorrect shebang:"
+    $ScriptsWithBrokenShebang | ForEach-Object {
+        Write-Host "##[error] '$_'"
+    }
+    Write-Host "`n`n##[error] Expected shebang for scripts in 'images/linux' folder is '$PatternUbuntu'"
+    Write-Host "##[error] Expected shebang for scripts in 'images/macos' folder is '$PatternMacOS'"
+    exit 1
+    else {
+        Write-Host "All scripts have correct shebang."
+    }
+}

images/linux/scripts/base/apt.sh

@@ -1,13 +1,23 @@
-#!/bin/bash
+#!/bin/bash -e
 
 export DEBIAN_FRONTEND=noninteractive
 apt-get -yq update
 apt-get -yq dist-upgrade
-systemctl disable apt-daily.service
+
+# Stop and disable apt-daily upgrade services;
+systemctl stop apt-daily.timer
 systemctl disable apt-daily.timer
+systemctl disable apt-daily.service
+systemctl stop apt-daily-upgrade.timer
 systemctl disable apt-daily-upgrade.timer
 systemctl disable apt-daily-upgrade.service
 
+# This step should completely disable any automatic updates except manual
+sudo sed -i 's/APT::Periodic::Update-Package-Lists "1"/APT::Periodic::Update-Package-Lists "0"/' /etc/apt/apt.conf.d/20auto-upgrades
+
+# Enable retry logic for apt up to 10 times
+echo "APT::Acquire::Retries \"10\";" > /etc/apt/apt.conf.d/80-retries 
+
 # Configure apt to always assume Y
 echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/90assumeyes
 

images/linux/scripts/base/repos.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 ################################################################################
 ##  File:  repos.sh
 ##  Desc:  Installs official Microsoft package repos for the distribution

images/linux/scripts/helpers/etc-environment.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 ################################################################################
 ##  File:  etc-environment.sh
 ##  Desc:  Helper functions for source and modify /etc/environment

images/linux/scripts/helpers/install.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 ################################################################################
 ##  File:  install.sh
 ##  Desc:  Helper functions for installing tools

images/linux/scripts/helpers/os.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 ################################################################################
 ##  File:  install-helpers.sh
 ##  Desc:  Helper functions for installing tools

images/linux/scripts/installers/azpowershell.sh

@@ -15,9 +15,13 @@ else
     versions=$(jq -r '.azureModules[] | select(.name | contains("az")) | .versions[]' $toolset)
 fi
 
+# Try to install and update PowerShellGet before the actual installation
+pwsh -Command "Install-Module -Name PowerShellGet -Force"
+pwsh -Command "Update-Module -Name PowerShellGet -Force"
+
 # Install Azure CLI (instructions taken from https://docs.microsoft.com/en-us/cli/azure/install-azure-cli)
 for version in ${versions[@]}; do
-    pwsh -Command "Save-Module -Name Az -LiteralPath /usr/share/az_$version -RequiredVersion $version -Force"
+    pwsh -Command "Save-Module -Name Az -LiteralPath /usr/share/az_$version -RequiredVersion $version -Force -Verbose"
 done
 
 # Run tests to determine that the software installed as expected

images/linux/scripts/installers/homebrew.sh

@@ -12,9 +12,6 @@ source $HELPER_SCRIPTS/etc-environment.sh
 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
 eval $(/home/linuxbrew/.linuxbrew/bin/brew shellenv)
 
-# Make brew files and directories writable by any user
-sudo chmod -R o+w $HOMEBREW_PREFIX
-
 # Update /etc/environemnt
 ## Put HOMEBREW_* variables
 brew shellenv|grep 'export HOMEBREW'|sed -E 's/^export (.*);$/\1/' | sudo tee -a /etc/environment

images/linux/scripts/installers/pipx-packages.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 ################################################################################
 ##  File:  pipx-packages.sh
 ##  Desc:  Install tools via pipx

images/linux/scripts/installers/post-deployment.sh

@@ -24,3 +24,7 @@ then
     echo "PATH = $PATH"
     exit 1
 fi
+
+# Clean yarn and npm cache
+yarn cache clean
+npm cache clean --force

images/macos/macos-11.0-Readme.md

@@ -1,6 +1,6 @@
 | Announcements |
 |-|
-| [[macOS] Built-in Python 2.7 will be used on macOS instead of Homebrew formula on November, 3rd.](https://github.com/actions/virtual-environments/issues/1848) |
+| [.NET 5.0 will become a default .NET version on November, 10](https://github.com/actions/virtual-environments/issues/1891) |
 | [macOS 11.0 (Big Sur) is available as a preview 🚀](https://github.com/actions/virtual-environments/issues/1814) |
 | [[macOS] Default Ruby version will be changed to 2.7 on October, 26](https://github.com/actions/virtual-environments/issues/1775) |
 | [Default Xcode will be changed to Xcode 12.0.1 on October, 20](https://github.com/actions/virtual-environments/issues/1712) |
@@ -9,7 +9,7 @@
 # macOS 11.0 info
 - System Version: macOS 11.0 (20A5395g)
 - Kernel Version: Darwin 20.1.0
-- Image Version: 20201020.1
+- Image Version: 20201024.1
 
 ## Installed Software
 ### Language and Runtime
@@ -34,14 +34,14 @@
 - Pip 20.2.3 (python 3.8)
 - Bundler version 2.1.4
 - Carthage 0.36.0
-- CocoaPods 1.9.3
+- CocoaPods 1.10.0
 - Homebrew 2.5.6
 - NPM 6.14.8
 - Yarn 1.22.5
 - NuGet 5.6.0.6489
 - Miniconda 4.8.3
 - RubyGems 3.1.4
-- Composer 1.10.15
+- Composer 2.0.1
 
 ### Project Management
 - Apache Maven 3.6.3
@@ -50,7 +50,7 @@
 
 ### Utilities
 - Curl 7.73.0
-- Git: 2.29.0
+- Git: 2.29.1
 - Git LFS: 2.12.0
 - GitHub CLI: 1.1.0
 - Hub CLI: 2.14.2
@@ -63,10 +63,10 @@
 - psql (PostgreSQL) 13.0
 - PostgreSQL 13.0
 - aria2 1.35.0
-- azcopy 10.6.0
+- azcopy 10.6.1
 - zstd 1.4.5
 - bazel 3.7.0
-- bazelisk 1.7.3
+- bazelisk 1.7.4
 - helm v3.3.4+ga61ce56
 - mongo v4.4.1
 - mongod v4.4.1
@@ -74,13 +74,13 @@
 - Newman 5.2.0
 
 ### Tools
-- Fastlane 2.164.0
+- Fastlane 2.165.0
 - Cmake 3.18.4
 - App Center CLI 2.7.2
 - Azure CLI 2.13.0
-- AWS CLI 2.0.57
+- AWS CLI 2.0.59
 - AWS SAM CLI 1.6.2
-- AWS Session Manager CLI 1.1.61.0
+- AWS Session Manager CLI 1.2.7.0
 - Aliyun CLI 3.0.60
 - GHCup v0.1.11
 - GHC 8.10.2
@@ -94,24 +94,24 @@
 ### Browsers
 - Safari 14.0.1 (16610.2.8.1.1)
 - SafariDriver 14.0.1 (16610.2.8.1.1)
-- Google Chrome 86.0.4240.80 
+- Google Chrome 86.0.4240.111 
 - ChromeDriver 86.0.4240.22
 - Microsoft Edge 85.0.564.70 
 - MSEdgeDriver 85.0.564.70
-- Mozilla Firefox 81.0.2
+- Mozilla Firefox 82.0
 - geckodriver 0.27.0
 
 ### Java
 | Version   | Vendor       | Environment Variable |
 | --------- | ------------ | -------------------- |
-| 1.8.0_265 | AdoptOpenJDK | JAVA_HOME_8_X64      |
+| 1.8.0_272 | AdoptOpenJDK | JAVA_HOME_8_X64      |
-| 11.0.8    | AdoptOpenJDK | JAVA_HOME_11_X64     |
+| 11.0.9    | AdoptOpenJDK | JAVA_HOME_11_X64     |
 ### Cached Tools
 #### Ruby
 - 2.4.10
 - 2.5.8
 - 2.6.6
-- 2.7.1
+- 2.7.2
 
 #### Python
 - 3.7.9
@@ -170,7 +170,6 @@
 | Version        | Build    | Path                         |
 | -------------- | -------- | ---------------------------- |
 | 12.2 (beta)    | 12B5035g | /Applications/Xcode_12.2.app |
-| 12.1           | 12A7403  | /Applications/Xcode_12.1.app |
 | 11.7 (default) | 11E801a  | /Applications/Xcode_11.7.app |
 
 #### Xcode Support Tools
@@ -180,40 +179,31 @@
 #### Installed SDKs
 | SDK                     | SDK Name             | Xcode Version |
 | ----------------------- | -------------------- | ------------- |
-| macOS 10.15             | macosx10.15          | 11.7, 12.1    |
+| macOS 10.15             | macosx10.15          | 11.7          |
 | macOS 11.0              | macosx11.0           | 12.2          |
 | iOS 13.7                | iphoneos13.7         | 11.7          |
-| iOS 14.1                | iphoneos14.1         | 12.1          |
 | iOS 14.2                | iphoneos14.2         | 12.2          |
 | Simulator - iOS 13.7    | iphonesimulator13.7  | 11.7          |
-| Simulator - iOS 14.1    | iphonesimulator14.1  | 12.1          |
 | Simulator - iOS 14.2    | iphonesimulator14.2  | 12.2          |
 | tvOS 13.4               | appletvos13.4        | 11.7          |
-| tvOS 14.0               | appletvos14.0        | 12.1          |
 | tvOS 14.2               | appletvos14.2        | 12.2          |
 | Simulator - tvOS 13.4   | appletvsimulator13.4 | 11.7          |
-| Simulator - tvOS 14.0   | appletvsimulator14.0 | 12.1          |
 | Simulator - tvOS 14.2   | appletvsimulator14.2 | 12.2          |
 | watchOS 6.2             | watchos6.2           | 11.7          |
-| watchOS 7.0             | watchos7.0           | 12.1          |
 | watchOS 7.1             | watchos7.1           | 12.2          |
 | Simulator - watchOS 6.2 | watchsimulator6.2    | 11.7          |
-| Simulator - watchOS 7.0 | watchsimulator7.0    | 12.1          |
 | Simulator - watchOS 7.1 | watchsimulator7.1    | 12.2          |
-| DriverKit 19.0          | driverkit.macosx19.0 | 11.7, 12.1    |
+| DriverKit 19.0          | driverkit.macosx19.0 | 11.7          |
 | DriverKit 20.0          | driverkit.macosx20.0 | 12.2          |
 
 #### Installed Simulators
 | OS          | Xcode Version | Simulators                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | ----------- | ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | iOS 13.7    | 11.7          | iPhone 11<br>iPhone 11 Pro<br>iPhone 11 Pro Max<br>iPhone 8<br>iPhone 8 Plus<br>iPhone SE (2nd generation)<br>iPad (7th generation)<br>iPad Air (3rd generation)<br>iPad Pro (11-inch) (2nd generation)<br>iPad Pro (12.9-inch) (4th generation)<br>iPad Pro (9.7-inch)                                                                                                                                                           |
-| iOS 14.1    | 12.1          | iPod touch (7th generation)<br>iPhone 11<br>iPhone 11 Pro<br>iPhone 11 Pro Max<br>iPhone 12<br>iPhone 12 mini<br>iPhone 12 Pro<br>iPhone 12 Pro Max<br>iPhone 8<br>iPhone 8 Plus<br>iPhone SE (2nd generation)<br>iPad (7th generation)<br>iPad (8th generation)<br>iPad Air (3rd generation)<br>iPad Air (4th generation)<br>iPad Pro (11-inch) (2nd generation)<br>iPad Pro (12.9-inch) (4th generation)<br>iPad Pro (9.7-inch) |
 | iOS 14.2    | 12.2          | iPod touch (7th generation)<br>iPhone 11<br>iPhone 11 Pro<br>iPhone 11 Pro Max<br>iPhone 12<br>iPhone 12 mini<br>iPhone 12 Pro<br>iPhone 12 Pro Max<br>iPhone 8<br>iPhone 8 Plus<br>iPhone SE (2nd generation)<br>iPad (7th generation)<br>iPad (8th generation)<br>iPad Air (3rd generation)<br>iPad Air (4th generation)<br>iPad Pro (11-inch) (2nd generation)<br>iPad Pro (12.9-inch) (4th generation)<br>iPad Pro (9.7-inch) |
 | tvOS 13.4   | 11.7          | Apple TV<br>Apple TV 4K<br>Apple TV 4K (at 1080p)                                                                                                                                                                                                                                                                                                                                                                                 |
-| tvOS 14.0   | 12.1          | Apple TV<br>Apple TV 4K<br>Apple TV 4K (at 1080p)                                                                                                                                                                                                                                                                                                                                                                                 |
 | tvOS 14.2   | 12.2          | Apple TV<br>Apple TV 4K<br>Apple TV 4K (at 1080p)                                                                                                                                                                                                                                                                                                                                                                                 |
 | watchOS 6.2 | 11.7          | Apple Watch Series 4 - 40mm<br>Apple Watch Series 4 - 44mm<br>Apple Watch Series 5 - 40mm<br>Apple Watch Series 5 - 44mm                                                                                                                                                                                                                                                                                                          |
-| watchOS 7.0 | 12.1          | Apple Watch Series 4 - 40mm<br>Apple Watch Series 4 - 44mm<br>Apple Watch Series 5 - 40mm<br>Apple Watch Series 5 - 44mm<br>Apple Watch Series 6 - 40mm<br>Apple Watch Series 6 - 44mm                                                                                                                                                                                                                                            |
 | watchOS 7.1 | 12.2          | Apple Watch Series 4 - 40mm<br>Apple Watch Series 4 - 44mm<br>Apple Watch Series 5 - 40mm<br>Apple Watch Series 5 - 44mm<br>Apple Watch Series 6 - 40mm<br>Apple Watch Series 6 - 44mm                                                                                                                                                                                                                                            |
 
 ### Android

images/macos/provision/bootstrap-provisioner/installNewProvisioner.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 BOOTSTRAP_PATH="$1"
 ProvisionerPackageUri="$2"
 ProvisionerScriptUri="$3"

images/macos/provision/configuration/finalize-vm.sh

@@ -20,6 +20,9 @@ cp $HOME/image-generation/output/software-report/systeminfo.txt $HOME/image-gene
 # we have to do that here because `npm install` is run in a few different places during image-generation
 npm cache clean --force
 
+# Clean yarn cache
+yarn cache clean
+
 # Clean up temporary directories
 rm -rf ~/utils ~/image-generation
 

images/macos/provision/core/chrome.sh

@@ -1,7 +1,8 @@
 #!/bin/bash -e -o pipefail
+source ~/utils/utils.sh
 
 echo "Installing Chrome..."
-brew cask install google-chrome
+brew_cask_install_ignoring_sha256 "google-chrome"
 
 echo "Installing Chrome Driver"
 brew cask install chromedriver

images/macos/provision/core/commonutils.sh

@@ -21,7 +21,6 @@ binst_common_utils=(
     gh
     p7zip
     ant
-    yamllint
     aria2
 )
 

images/macos/provision/core/edge.sh

@@ -3,11 +3,7 @@
 source ~/utils/utils.sh
 
 echo "Installing Microsoft Edge..."
-# Workaround to install version 85 since webdriver is broken for 86
-cd "$(brew --repo homebrew/homebrew-cask)"
-git checkout 81f9d08d2b9b7557c0178621078cf59d2c5db2bc
 brew cask install microsoft-edge
-git checkout master
 
 EDGE_INSTALLATION_PATH="/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"
 EDGE_VERSION=$("$EDGE_INSTALLATION_PATH" --version | cut -d' ' -f 3)

images/macos/provision/core/pipx-packages.sh

@@ -0,0 +1,18 @@
+source ~/utils/utils.sh
+
+export PATH="$PATH:/opt/pipx_bin"
+
+toolset=$(get_toolset_path)
+pipx_packages=$(jq -r ".pipx[] .package" $toolset)
+
+for package in $pipx_packages; do
+    python_version=$(jq -r ".pipx[] | select(.package == \"$package\") .python" $toolset)
+    if [ "$python_version" != "null" ]; then
+        python_path="$HOME/hostedtoolcache/Python/$python_version*/x64/bin/python$python_version"
+        echo "Install $package into python $python_path"
+        pipx install $package --python $python_path
+    else
+        echo "Install $package into default python"
+        pipx install $package
+    fi
+done

images/macos/provision/core/python.sh

@@ -17,3 +17,13 @@ echo "Brew Installing Python 2"
 brew tap-new --no-git local/python2
 FORMULA_PATH=$(brew extract python@2 local/python2 | grep "Homebrew/Library/Taps")
 brew install $FORMULA_PATH
+
+echo "Installing pipx"
+export PIPX_BIN_DIR=/usr/local/opt/pipx_bin
+export PIPX_HOME=/usr/local/opt/pipx
+
+brew install pipx
+
+echo "export PIPX_BIN_DIR=${PIPX_BIN_DIR}" >> "${HOME}/.bashrc"
+echo "export PIPX_HOME=${PIPX_HOME}" >> "${HOME}/.bashrc"
+echo 'export PATH="$PIPX_BIN_DIR:$PATH"' >> "${HOME}/.bashrc"

images/macos/provision/core/xamarin.sh

@@ -9,6 +9,7 @@ XAMARIN_MAC_VERSIONS=($(get_toolset_value '.xamarin."mac-versions" | reverse | .
 XAMARIN_ANDROID_VERSIONS=($(get_toolset_value '.xamarin."android-versions" | reverse | .[]'))
 LATEST_SDK_SYMLINK=$(get_toolset_value '.xamarin.bundles[0].symlink')
 CURRENT_SDK_SYMLINK=$(get_toolset_value '.xamarin."bundle-default"')
+DEFAULT_XCODE_VERSION=$(get_default_xcode_from_toolset)
 
 if [ "$CURRENT_SDK_SYMLINK" == "latest" ]; then
   CURRENT_SDK_SYMLINK=$LATEST_SDK_SYMLINK
@@ -82,3 +83,8 @@ popd
 
 echo "Clean up packages..."
 sudo rm -rf "$TMPMOUNT"
+
+# Fix Xamarin issue with Xcode symlink: https://github.com/xamarin/xamarin-macios/issues/9960
+PREFERENCES_XAMARIN_DIR="${HOME}/Library/Preferences/Xamarin"
+mkdir -p $PREFERENCES_XAMARIN_DIR
+/usr/libexec/PlistBuddy -c "add :AppleSdkRoot string /Applications/Xcode_${DEFAULT_XCODE_VERSION}.app" $PREFERENCES_XAMARIN_DIR/Settings.plist

images/macos/provision/utils/utils.sh

@@ -108,3 +108,16 @@ verlte() {
     sortedVersion=$(echo -e "$1\n$2" | sort -V | head -n1)
     [  "$1" = "$sortedVersion" ]
 }
+
+brew_cask_install_ignoring_sha256() {
+    local TOOL_NAME=$1
+
+    CASK_DIR="$(brew --repo homebrew/cask)/Casks"
+    chmod a+w "$CASK_DIR/$TOOL_NAME.rb"
+    SHA=$(grep "sha256" "$CASK_DIR/$TOOL_NAME.rb" | awk '{print $2}')
+    sed -i '' "s/$SHA/:no_check/" "$CASK_DIR/$TOOL_NAME.rb"
+    brew cask install $TOOL_NAME
+    pushd $CASK_DIR
+    git checkout HEAD -- "$TOOL_NAME.rb"
+    popd
+}

images/macos/software-report/SoftwareReport.Common.psm1

@@ -96,6 +96,11 @@ function Get-PipVersion {
     return "${versionPart1} ${versionPart2} ${versionPart3}"
 }
 
+function Get-PipxVersion {
+    $pipxVersion = Run-Command "pipx --version" -SuppressStderr
+    return "Pipx $pipxVersion"
+}
+
 function Get-NVMNodeVersionList {
     $nvmPath = Join-Path $env:HOME ".nvm" "nvm.sh"
     $nvmInitCommand = ". ${nvmPath} > /dev/null 2>&1 || true"

images/macos/software-report/SoftwareReport.Generator.ps1

@@ -83,6 +83,7 @@ $npmVersion = Run-Command "npm --version"
 $yarnVersion = Run-Command "yarn --version"
 $nugetVersion = Run-Command "nuget help" | Select-Object -First 1 | Take-Part -Part 2
 $pip3Version = Get-PipVersion -Version 3
+$pipxVersion = Get-PipxVersion
 $condaVersion = Invoke-Expression "conda --version"
 $rubyGemsVersion = Run-Command "gem --version"
 $composerVersion = Run-Command "composer --version" | Take-Part -Part 2
@@ -99,6 +100,7 @@ if ($os.IsLessThanBigSur) {
 
 $markdown += New-MDList -Style Unordered -Lines @(
     "Pip ${pip3Version}",
+    $pipxVersion,
     $bundlerVersion,
     "Carthage ${carthageVersion}",
     "CocoaPods ${cocoaPodsVersion}",

images/macos/templates/macOS-10.13.json

@@ -181,7 +181,8 @@
                 "./provision/core/edge.sh",
                 "./provision/core/firefox.sh",
                 "./provision/core/toolcache-high-sierra.sh",
-                "./provision/core/pypy.sh"
+                "./provision/core/pypy.sh",
+                "./provision/core/pipx-packages.sh"
             ]
         },
         {

images/macos/templates/macOS-10.14.json

@@ -186,7 +186,8 @@
                 "./provision/core/miniconda.sh",
                 "./provision/core/xcode-postbuild.sh",
                 "./provision/core/toolcache.sh",
-                "./provision/core/pypy.sh"
+                "./provision/core/pypy.sh",
+                "./provision/core/pipx-packages.sh"
             ],
             "environment_vars": [
                 "GITHUB_FEED_TOKEN={{user `github_feed_token`}}"

images/macos/templates/macOS-10.15.json

@@ -185,7 +185,8 @@
                 "./provision/core/firefox.sh",
                 "./provision/core/xcode-postbuild.sh",
                 "./provision/core/toolcache.sh",
-                "./provision/core/pypy.sh"
+                "./provision/core/pypy.sh",
+                "./provision/core/pipx-packages.sh"
             ],
             "environment_vars": [
                 "GITHUB_FEED_TOKEN={{user `github_feed_token`}}"

images/macos/templates/macOS-11.0.json

@@ -181,7 +181,8 @@
                 "./provision/core/chrome.sh",
                 "./provision/core/edge.sh",
                 "./provision/core/firefox.sh",
-                "./provision/core/toolcache.sh"
+                "./provision/core/toolcache.sh",
+                "./provision/core/pipx-packages.sh"
             ],
             "environment_vars": [
                 "GITHUB_FEED_TOKEN={{user `github_feed_token`}}"

images/macos/tests/Linters.Tests.ps1

@@ -3,10 +3,6 @@ Import-Module "$PSScriptRoot/../helpers/Tests.Helpers.psm1"
 $os = Get-OSVersion
 
 Describe "Linters" {
-    It "yamllint" {
-        "yamllint --version" | Should -ReturnZeroExitCode
-    }
-
     It "SwiftLint" -Skip:($os.IsHighSierra) {
         "swiftlint version" | Should -ReturnZeroExitCode
     }

images/macos/tests/PipxPackages.Tests.ps1

@@ -0,0 +1,7 @@
+Describe "PipxPackages" {
+    $pipxToolset = Get-ToolsetValue "pipx"
+    $testCases = $pipxToolset | ForEach-Object { @{package = $_.package; cmd = $_.cmd} }
+    It "<package>" -TestCases $testCases {
+        "$cmd" | Should -ReturnZeroExitCode
+    }
+}

images/macos/tests/Python.Tests.ps1

@@ -32,6 +32,10 @@ Describe "Python" {
         "pip3 --version" | Should -ReturnZeroExitCode
     }
 
+    It "Pipx is available" {
+        "pipx --version" | Should -ReturnZeroExitCode
+    }
+
     It "Pip 3 and Python 3 came from the same brew formula" {
         $pip3Path = Split-Path (readlink (which pip3))
         $python3Path = Split-Path (readlink (which python3))

images/macos/toolsets/toolset-10.13.json

@@ -194,5 +194,11 @@
                 "3.6"
             ]
         }
+    ],
+    "pipx": [
+        {
+            "package": "yamllint",
+            "cmd": "yamllint --version"
+        }
     ]
 }

images/macos/toolsets/toolset-10.14.json

@@ -254,5 +254,11 @@
                 "1.15.*"
             ]
         }
+    ],
+    "pipx": [
+        {
+            "package": "yamllint",
+            "cmd": "yamllint --version"
+        }
     ]
 }

images/macos/toolsets/toolset-10.15.json

@@ -163,5 +163,11 @@
                 "1.15.*"
             ]
         }
+    ],
+    "pipx": [
+        {
+            "package": "yamllint",
+            "cmd": "yamllint --version"
+        }
     ]
 }

images/macos/toolsets/toolset-11.0.json

@@ -88,5 +88,11 @@
                 "1.15.*"
             ]
         }
+    ],
+    "pipx": [
+        {
+            "package": "yamllint",
+            "cmd": "yamllint --version"
+        }
     ]
 }

images/win/scripts/Installers/Initialize-VM.ps1

@@ -128,11 +128,7 @@ if (Test-IsWin19) {
 }
 
 # Expand disk size of OS drive
-New-Item -Path d:\ -Name cmds.txt -ItemType File -Force
+$driveLetter = "C"
-Add-Content -Path d:\cmds.txt "SELECT VOLUME=C`r`nEXTEND"
+$size = Get-PartitionSupportedSize -DriveLetter $driveLetter
-
+Resize-Partition -DriveLetter $driveLetter -Size $size.SizeMax
-$expandResult = (diskpart /s 'd:\cmds.txt')
+Get-Volume | Select-Object DriveLetter, SizeRemaining, Size | Sort-Object DriveLetter
-Write-Host $expandResult
-
-Write-Host "Disk sizes after expansion"
-wmic logicaldisk get size,freespace,caption

images/win/windows2019.json

@@ -13,7 +13,7 @@
         "virtual_network_resource_group_name": "{{env `VNET_RESOURCE_GROUP`}}",
         "virtual_network_subnet_name": "{{env `VNET_SUBNET`}}",
         "private_virtual_network_with_public_ip": "{{env `PRIVATE_VIRTUAL_NETWORK_WITH_PUBLIC_IP`}}",
-        "vm_size": "Standard_D4_v2",
+        "vm_size": "Standard_DS4_v2",
         "run_scan_antivirus": "false",
         "root_folder": "C:",
         "toolset_json_path": "{{env `TEMP`}}\\toolset.json",