diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index b95d5ebc2..cf1a53405 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -17,10 +17,11 @@ A clear and concise description of what the bug is, and why you consider it to b **Virtual environments affected** +- [ ] Ubuntu 16.04 +- [ ] Ubuntu 18.04 +- [ ] Ubuntu 20.04 - [ ] macOS 10.15 -- [ ] Ubuntu 16.04 LTS -- [ ] Ubuntu 18.04 LTS -- [ ] Ubuntu 20.04 LTS +- [ ] macOS 11.0 - [ ] Windows Server 2016 R2 - [ ] Windows Server 2019 diff --git a/.github/ISSUE_TEMPLATE/tool-request.md b/.github/ISSUE_TEMPLATE/tool-request.md index 2eeb87378..72fdd0064 100644 --- a/.github/ISSUE_TEMPLATE/tool-request.md +++ b/.github/ISSUE_TEMPLATE/tool-request.md @@ -25,10 +25,11 @@ assignees: '' **Virtual environments affected** +- [ ] Ubuntu 16.04 +- [ ] Ubuntu 18.04 +- [ ] Ubuntu 20.04 - [ ] macOS 10.15 -- [ ] Ubuntu 16.04 LTS -- [ ] Ubuntu 18.04 LTS -- [ ] Ubuntu 20.04 LTS +- [ ] macOS 11.0 - [ ] Windows Server 2016 R2 - [ ] Windows Server 2019 diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 42eb7cad8..48ec7dd65 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -1,11 +1,11 @@ # CI Validation -name: CI +name: Linter on: pull_request: - branches: [$default-branch] - + branches: [ main ] + jobs: build: name: Lint JSON & MD files @@ -23,3 +23,7 @@ jobs: VALIDATE_JSON: true VALIDATE_MD: true DEFAULT_BRANCH: ${{ github.base_ref }} + + - name: Checking shebang lines in MacOS and Ubuntu releases. + run: ./images.CI/shebang-linter.ps1 + shell: pwsh diff --git a/README.md b/README.md index b3c596491..9456f0bc0 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,18 @@ For general questions about using the virtual environments or writing your Actio | Ubuntu 20.04 | `ubuntu-20.04` | [ubuntu-20.04] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu20&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu20&redirect=1) | Ubuntu 18.04 | `ubuntu-latest` or `ubuntu-18.04` | [ubuntu-18.04] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu18&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu18&redirect=1) | Ubuntu 16.04 | `ubuntu-16.04` | [ubuntu-16.04] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu16&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu16&redirect=1) | -| macOS 10.15 | `macos-latest` or `macos-10.15` | [macOS-10.15] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=MacOS&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=MacOS&redirect=1) +| macOS 11.0 | `macos-11.0` | [macOS-11.0] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=macos-11.0&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=macos-11.0&redirect=1) +| macOS 10.15 | `macos-latest` or `macos-10.15` | [macOS-10.15] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=macos-10.15&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=macos-10.15&redirect=1) | Windows Server 2019 | `windows-latest` or `windows-2019` | [windows-2019] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2019&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2019&redirect=1) | Windows Server 2016 | `windows-2016` | [windows-2016] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2016&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2016&redirect=1) ``` The Ubuntu 20.04 virtual environment is currently provided as a preview only. The "ubuntu-latest" YAML workflow label still uses the Ubuntu 18.04 virtual environment. ``` +``` +The MacOS 11.0 virtual environment is currently provided as a preview only. +The "macos-latest" YAML workflow label still uses the MacOS 10.15 virtual environment. +``` ***Looking for other Linux distributions?*** We do not plan to offer other Linux distributions. We recommend using Docker if you'd like to build using other distributions with the hosted virtual environments. Alternatively, you can leverage [self-hosted runners] and fully customize your environment to your needs. @@ -56,4 +61,5 @@ Low Impact changes will be pinned in this repository and marked with the [Announ [Windows-2019]: https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md [windows-2016]: https://github.com/actions/virtual-environments/blob/main/images/win/Windows2016-Readme.md [macOS-10.15]: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md +[macOS-11.0]: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11.0-Readme.md [self-hosted runners]: https://help.github.com/en/actions/hosting-your-own-runners diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 89216e216..60837bc1a 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -52,7 +52,8 @@ jobs: inputs: targetType: 'filePath' filePath: ./images.CI/macos/select-datastore.ps1 - arguments: -VIServer "$(vcenter-server-v2)" ` + arguments: -VMName "$(VirtualMachineName)" ` + -VIServer "$(vcenter-server-v2)" ` -VIUserName "$(vcenter-username-v2)" ` -VIPassword "$(vcenter-password-v2)" @@ -121,8 +122,8 @@ jobs: condition: always() - task: PowerShell@2 - displayName: 'Move vm to cold storage' - condition: succeededOrFailed() + displayName: 'Move vm to cold storage and clear datastore tag' + condition: always() inputs: targetType: 'filePath' filePath: ./images.CI/macos/move-vm.ps1 diff --git a/images.CI/macos/move-vm.ps1 b/images.CI/macos/move-vm.ps1 index 698721409..37e839066 100644 --- a/images.CI/macos/move-vm.ps1 +++ b/images.CI/macos/move-vm.ps1 @@ -48,6 +48,13 @@ Import-Module $PSScriptRoot\helpers.psm1 -DisableNameChecking # Connection to a vCenter Server system Connect-VCServer +# Clear previously assigned tag with VM Name +try { + Remove-Tag $VMName -Confirm:$false +} catch { + Write-Host "Tag with $VMName doesn't exist" +} + $vm = Get-VM $VMName if ($env:AGENT_JOBSTATUS -eq 'Failed') { diff --git a/images.CI/macos/select-datastore.ps1 b/images.CI/macos/select-datastore.ps1 index d6477700c..2884fdf46 100644 --- a/images.CI/macos/select-datastore.ps1 +++ b/images.CI/macos/select-datastore.ps1 @@ -20,6 +20,10 @@ vCenter password (Example "12345678") [CmdletBinding()] param( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$VMName, + [Parameter(Mandatory)] [ValidateNotNullOrEmpty()] [string]$VIServer, @@ -30,35 +34,72 @@ param( [Parameter(Mandatory)] [ValidateNotNullOrEmpty()] - [string]$VIPassword + [string]$VIPassword, + + [string]$TagCategory = "Busy" ) # Import helpers module Import-Module $PSScriptRoot\helpers.psm1 -DisableNameChecking +function Select-DataStore { + param ( + [string]$VMName, + [string]$TagCategory, + [string]$TemplateDatastore = "ds-local-Datastore-*", + [int]$ThresholdInGb = 400, + [int]$VMCount = 2, + [int]$Retries = 5 + ) + + # 1. Name starts with ds-local-Datastore + # 2. FreespaceGB > 400 Gb + # 3. VM count on a datastore < 2 + + Write-Host "Start Datastore selection process..." + $allDatastores = Get-Datastore -Name $templateDatastore | Where-Object { $_.State -eq "Available" } + $buildDatastore = $allDatastores | Where-Object { $_.FreeSpaceGB -ge $thresholdInGb } | Where-Object { + $vmOnDatastore = @((Get-ChildItem -Path $_.DatastoreBrowserPath).Name -notmatch "^\.").Count + $vmOnDatastore -lt $vmCount + } | Select-Object -ExpandProperty Name -First 1 + + $tag = Get-Tag -Category $TagCategory -Name $VMName -ErrorAction Ignore + if (-not $tag) + { + $tag = New-Tag -Name $VMName -Category $TagCategory + } + + New-TagAssignment -Tag $tag -Entity $buildDatastore | Out-Null + + # Wait for 60 seconds to check if any other tags are assigned to the same datastore + Start-Sleep -Seconds 60 + # Take only first 2 tags, all the others will go to the next round + $tagAssignments = (Get-TagAssignment -Entity $buildDatastore).Tag.Name | Select-Object -First 2 + $isAllow = $tagAssignments -contains $VMName + + if ($isAllow) + { + Write-Host "Datastore selected successfully" + Write-Host "##vso[task.setvariable variable=buildDatastore;issecret=true]$buildDatastore" + return + } + + # Remove the tag if datastore wasn't selected + Remove-Tag $tag -Confirm:$false + + $retries-- + if ($retries -le 0) + { + Write-Host "##vso[task.LogIssue type=error;]No datastores found for the condition" + exit 1 + } + + Write-Host "Datastore select failed, $retries left" + Select-DataStore -VMName $VMName -TagCategory $TagCategory -Retries $retries +} + # Connection to a vCenter Server system Connect-VCServer # Get a target datastore for current deployment -# 1. Name starts with ds-local-Datastore -# 2. FreespaceGB > 400 Gb -# 3. VM count on a datastore < 2 -$templateDatastore = "ds-local-Datastore-*" -$thresholdInGb = 400 -$vmCount = 2 -$allDatastores = Get-Datastore -Name $templateDatastore | Where-Object { $_.State -eq "Available" } -$buildDatastore = $allDatastores | Where-Object { $_.FreeSpaceGB -ge $thresholdInGb } | Where-Object { - $vmOnDatastore = @((Get-ChildItem -Path $_.DatastoreBrowserPath).Name -notmatch "^\.").Count - $vmOnDatastore -lt $vmCount - } | Select-Object -ExpandProperty Name -First 1 - -if ($buildDatastore) -{ - Write-Host "Datastore selected successfully" - Write-Host "##vso[task.setvariable variable=buildDatastore;issecret=true]$buildDatastore" -} -else -{ - Write-Host "##vso[task.LogIssue type=error;]No datastores found for the condition" - exit 1 -} \ No newline at end of file +Select-DataStore -VMName $VMName -TagCategory $TagCategory diff --git a/images.CI/shebang-linter.ps1 b/images.CI/shebang-linter.ps1 new file mode 100644 index 000000000..a3862f658 --- /dev/null +++ b/images.CI/shebang-linter.ps1 @@ -0,0 +1,43 @@ +$ErrorActionPreference = "Stop" + +function Validate-Scripts { + Param ( + [Parameter(Mandatory=$true)] + [string[]]$Path, + [Parameter(Mandatory=$true)] + [string]$ExpectedShebang + ) + $ScriptWithoutShebangLine = @() + Get-ChildItem $path -Recurse -File -Filter "*.sh" | ForEach-Object { + $relativePath = Resolve-Path $_.FullName -Relative + $shebangLine = Get-Content -Path $_.FullName | Select-Object -First 1 + if ($shebangLine -eq $ExpectedShebang) { + Write-Host "[+] '$relativePath'" + } + else { + Write-Host "[-] '$relativePath'" + $ScriptWithoutShebangLine += $relativePath + } + } + return $ScriptWithoutShebangLine +} + +$PathUbuntu = "./images/linux/scripts" +$PathMacOS = "./images/macos/provision" +$PatternUbuntu = "#!/bin/bash -e" +$PatternMacOS = "#!/bin/bash -e -o pipefail" +$ScriptsWithBrokenShebang = @() +$ScriptsWithBrokenShebang += Validate-Scripts -Path $PathUbuntu -ExpectedShebang $PatternUbuntu +$ScriptsWithBrokenShebang += Validate-Scripts -Path $PathMacOS -ExpectedShebang $PatternMacOS +if ($ScriptsWithBrokenShebang.Length -gt 0) { + Write-Host "`n`n`n##[error] The following scripts have incorrect shebang:" + $ScriptsWithBrokenShebang | ForEach-Object { + Write-Host "##[error] '$_'" + } + Write-Host "`n`n##[error] Expected shebang for scripts in 'images/linux' folder is '$PatternUbuntu'" + Write-Host "##[error] Expected shebang for scripts in 'images/macos' folder is '$PatternMacOS'" + exit 1 + else { + Write-Host "All scripts have correct shebang." + } +} \ No newline at end of file diff --git a/images/linux/scripts/base/apt.sh b/images/linux/scripts/base/apt.sh index 8afcbe8d2..1efbb6721 100644 --- a/images/linux/scripts/base/apt.sh +++ b/images/linux/scripts/base/apt.sh @@ -1,13 +1,23 @@ -#!/bin/bash +#!/bin/bash -e export DEBIAN_FRONTEND=noninteractive apt-get -yq update apt-get -yq dist-upgrade -systemctl disable apt-daily.service + +# Stop and disable apt-daily upgrade services; +systemctl stop apt-daily.timer systemctl disable apt-daily.timer +systemctl disable apt-daily.service +systemctl stop apt-daily-upgrade.timer systemctl disable apt-daily-upgrade.timer systemctl disable apt-daily-upgrade.service +# This step should completely disable any automatic updates except manual +sudo sed -i 's/APT::Periodic::Update-Package-Lists "1"/APT::Periodic::Update-Package-Lists "0"/' /etc/apt/apt.conf.d/20auto-upgrades + +# Enable retry logic for apt up to 10 times +echo "APT::Acquire::Retries \"10\";" > /etc/apt/apt.conf.d/80-retries + # Configure apt to always assume Y echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/90assumeyes diff --git a/images/linux/scripts/base/repos.sh b/images/linux/scripts/base/repos.sh index 4ecd2028c..9005d32aa 100644 --- a/images/linux/scripts/base/repos.sh +++ b/images/linux/scripts/base/repos.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e ################################################################################ ## File: repos.sh ## Desc: Installs official Microsoft package repos for the distribution diff --git a/images/linux/scripts/helpers/etc-environment.sh b/images/linux/scripts/helpers/etc-environment.sh index 24801723e..eba0ecc05 100644 --- a/images/linux/scripts/helpers/etc-environment.sh +++ b/images/linux/scripts/helpers/etc-environment.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e ################################################################################ ## File: etc-environment.sh ## Desc: Helper functions for source and modify /etc/environment diff --git a/images/linux/scripts/helpers/install.sh b/images/linux/scripts/helpers/install.sh index bb1befe22..e42cd3194 100644 --- a/images/linux/scripts/helpers/install.sh +++ b/images/linux/scripts/helpers/install.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e ################################################################################ ## File: install.sh ## Desc: Helper functions for installing tools diff --git a/images/linux/scripts/helpers/os.sh b/images/linux/scripts/helpers/os.sh index 9c30c811a..8bec86080 100644 --- a/images/linux/scripts/helpers/os.sh +++ b/images/linux/scripts/helpers/os.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e ################################################################################ ## File: install-helpers.sh ## Desc: Helper functions for installing tools diff --git a/images/linux/scripts/installers/azpowershell.sh b/images/linux/scripts/installers/azpowershell.sh index a8488af7f..47ac5ff3e 100644 --- a/images/linux/scripts/installers/azpowershell.sh +++ b/images/linux/scripts/installers/azpowershell.sh @@ -15,9 +15,13 @@ else versions=$(jq -r '.azureModules[] | select(.name | contains("az")) | .versions[]' $toolset) fi +# Try to install and update PowerShellGet before the actual installation +pwsh -Command "Install-Module -Name PowerShellGet -Force" +pwsh -Command "Update-Module -Name PowerShellGet -Force" + # Install Azure CLI (instructions taken from https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) for version in ${versions[@]}; do - pwsh -Command "Save-Module -Name Az -LiteralPath /usr/share/az_$version -RequiredVersion $version -Force" + pwsh -Command "Save-Module -Name Az -LiteralPath /usr/share/az_$version -RequiredVersion $version -Force -Verbose" done # Run tests to determine that the software installed as expected diff --git a/images/linux/scripts/installers/homebrew.sh b/images/linux/scripts/installers/homebrew.sh index 865923f25..dd9e8f9e5 100644 --- a/images/linux/scripts/installers/homebrew.sh +++ b/images/linux/scripts/installers/homebrew.sh @@ -12,9 +12,6 @@ source $HELPER_SCRIPTS/etc-environment.sh /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" eval $(/home/linuxbrew/.linuxbrew/bin/brew shellenv) -# Make brew files and directories writable by any user -sudo chmod -R o+w $HOMEBREW_PREFIX - # Update /etc/environemnt ## Put HOMEBREW_* variables brew shellenv|grep 'export HOMEBREW'|sed -E 's/^export (.*);$/\1/' | sudo tee -a /etc/environment diff --git a/images/linux/scripts/installers/pipx-packages.sh b/images/linux/scripts/installers/pipx-packages.sh index 07e1427e3..3b625fe41 100644 --- a/images/linux/scripts/installers/pipx-packages.sh +++ b/images/linux/scripts/installers/pipx-packages.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e ################################################################################ ## File: pipx-packages.sh ## Desc: Install tools via pipx diff --git a/images/linux/scripts/installers/post-deployment.sh b/images/linux/scripts/installers/post-deployment.sh index 95a2e2fa9..d427a172e 100644 --- a/images/linux/scripts/installers/post-deployment.sh +++ b/images/linux/scripts/installers/post-deployment.sh @@ -24,3 +24,7 @@ then echo "PATH = $PATH" exit 1 fi + +# Clean yarn and npm cache +yarn cache clean +npm cache clean --force \ No newline at end of file diff --git a/images/macos/macos-11.0-Readme.md b/images/macos/macos-11.0-Readme.md index b4776beb0..75189112d 100644 --- a/images/macos/macos-11.0-Readme.md +++ b/images/macos/macos-11.0-Readme.md @@ -1,6 +1,6 @@ | Announcements | |-| -| [[macOS] Built-in Python 2.7 will be used on macOS instead of Homebrew formula on November, 3rd.](https://github.com/actions/virtual-environments/issues/1848) | +| [.NET 5.0 will become a default .NET version on November, 10](https://github.com/actions/virtual-environments/issues/1891) | | [macOS 11.0 (Big Sur) is available as a preview 🚀](https://github.com/actions/virtual-environments/issues/1814) | | [[macOS] Default Ruby version will be changed to 2.7 on October, 26](https://github.com/actions/virtual-environments/issues/1775) | | [Default Xcode will be changed to Xcode 12.0.1 on October, 20](https://github.com/actions/virtual-environments/issues/1712) | @@ -9,7 +9,7 @@ # macOS 11.0 info - System Version: macOS 11.0 (20A5395g) - Kernel Version: Darwin 20.1.0 -- Image Version: 20201020.1 +- Image Version: 20201024.1 ## Installed Software ### Language and Runtime @@ -34,14 +34,14 @@ - Pip 20.2.3 (python 3.8) - Bundler version 2.1.4 - Carthage 0.36.0 -- CocoaPods 1.9.3 +- CocoaPods 1.10.0 - Homebrew 2.5.6 - NPM 6.14.8 - Yarn 1.22.5 - NuGet 5.6.0.6489 - Miniconda 4.8.3 - RubyGems 3.1.4 -- Composer 1.10.15 +- Composer 2.0.1 ### Project Management - Apache Maven 3.6.3 @@ -50,7 +50,7 @@ ### Utilities - Curl 7.73.0 -- Git: 2.29.0 +- Git: 2.29.1 - Git LFS: 2.12.0 - GitHub CLI: 1.1.0 - Hub CLI: 2.14.2 @@ -63,10 +63,10 @@ - psql (PostgreSQL) 13.0 - PostgreSQL 13.0 - aria2 1.35.0 -- azcopy 10.6.0 +- azcopy 10.6.1 - zstd 1.4.5 - bazel 3.7.0 -- bazelisk 1.7.3 +- bazelisk 1.7.4 - helm v3.3.4+ga61ce56 - mongo v4.4.1 - mongod v4.4.1 @@ -74,13 +74,13 @@ - Newman 5.2.0 ### Tools -- Fastlane 2.164.0 +- Fastlane 2.165.0 - Cmake 3.18.4 - App Center CLI 2.7.2 - Azure CLI 2.13.0 -- AWS CLI 2.0.57 +- AWS CLI 2.0.59 - AWS SAM CLI 1.6.2 -- AWS Session Manager CLI 1.1.61.0 +- AWS Session Manager CLI 1.2.7.0 - Aliyun CLI 3.0.60 - GHCup v0.1.11 - GHC 8.10.2 @@ -94,24 +94,24 @@ ### Browsers - Safari 14.0.1 (16610.2.8.1.1) - SafariDriver 14.0.1 (16610.2.8.1.1) -- Google Chrome 86.0.4240.80 +- Google Chrome 86.0.4240.111 - ChromeDriver 86.0.4240.22 - Microsoft Edge 85.0.564.70 - MSEdgeDriver 85.0.564.70 -- Mozilla Firefox 81.0.2 +- Mozilla Firefox 82.0 - geckodriver 0.27.0 ### Java | Version | Vendor | Environment Variable | | --------- | ------------ | -------------------- | -| 1.8.0_265 | AdoptOpenJDK | JAVA_HOME_8_X64 | -| 11.0.8 | AdoptOpenJDK | JAVA_HOME_11_X64 | +| 1.8.0_272 | AdoptOpenJDK | JAVA_HOME_8_X64 | +| 11.0.9 | AdoptOpenJDK | JAVA_HOME_11_X64 | ### Cached Tools #### Ruby - 2.4.10 - 2.5.8 - 2.6.6 -- 2.7.1 +- 2.7.2 #### Python - 3.7.9 @@ -170,7 +170,6 @@ | Version | Build | Path | | -------------- | -------- | ---------------------------- | | 12.2 (beta) | 12B5035g | /Applications/Xcode_12.2.app | -| 12.1 | 12A7403 | /Applications/Xcode_12.1.app | | 11.7 (default) | 11E801a | /Applications/Xcode_11.7.app | #### Xcode Support Tools @@ -180,40 +179,31 @@ #### Installed SDKs | SDK | SDK Name | Xcode Version | | ----------------------- | -------------------- | ------------- | -| macOS 10.15 | macosx10.15 | 11.7, 12.1 | +| macOS 10.15 | macosx10.15 | 11.7 | | macOS 11.0 | macosx11.0 | 12.2 | | iOS 13.7 | iphoneos13.7 | 11.7 | -| iOS 14.1 | iphoneos14.1 | 12.1 | | iOS 14.2 | iphoneos14.2 | 12.2 | | Simulator - iOS 13.7 | iphonesimulator13.7 | 11.7 | -| Simulator - iOS 14.1 | iphonesimulator14.1 | 12.1 | | Simulator - iOS 14.2 | iphonesimulator14.2 | 12.2 | | tvOS 13.4 | appletvos13.4 | 11.7 | -| tvOS 14.0 | appletvos14.0 | 12.1 | | tvOS 14.2 | appletvos14.2 | 12.2 | | Simulator - tvOS 13.4 | appletvsimulator13.4 | 11.7 | -| Simulator - tvOS 14.0 | appletvsimulator14.0 | 12.1 | | Simulator - tvOS 14.2 | appletvsimulator14.2 | 12.2 | | watchOS 6.2 | watchos6.2 | 11.7 | -| watchOS 7.0 | watchos7.0 | 12.1 | | watchOS 7.1 | watchos7.1 | 12.2 | | Simulator - watchOS 6.2 | watchsimulator6.2 | 11.7 | -| Simulator - watchOS 7.0 | watchsimulator7.0 | 12.1 | | Simulator - watchOS 7.1 | watchsimulator7.1 | 12.2 | -| DriverKit 19.0 | driverkit.macosx19.0 | 11.7, 12.1 | +| DriverKit 19.0 | driverkit.macosx19.0 | 11.7 | | DriverKit 20.0 | driverkit.macosx20.0 | 12.2 | #### Installed Simulators | OS | Xcode Version | Simulators | | ----------- | ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | iOS 13.7 | 11.7 | iPhone 11
iPhone 11 Pro
iPhone 11 Pro Max
iPhone 8
iPhone 8 Plus
iPhone SE (2nd generation)
iPad (7th generation)
iPad Air (3rd generation)
iPad Pro (11-inch) (2nd generation)
iPad Pro (12.9-inch) (4th generation)
iPad Pro (9.7-inch) | -| iOS 14.1 | 12.1 | iPod touch (7th generation)
iPhone 11
iPhone 11 Pro
iPhone 11 Pro Max
iPhone 12
iPhone 12 mini
iPhone 12 Pro
iPhone 12 Pro Max
iPhone 8
iPhone 8 Plus
iPhone SE (2nd generation)
iPad (7th generation)
iPad (8th generation)
iPad Air (3rd generation)
iPad Air (4th generation)
iPad Pro (11-inch) (2nd generation)
iPad Pro (12.9-inch) (4th generation)
iPad Pro (9.7-inch) | | iOS 14.2 | 12.2 | iPod touch (7th generation)
iPhone 11
iPhone 11 Pro
iPhone 11 Pro Max
iPhone 12
iPhone 12 mini
iPhone 12 Pro
iPhone 12 Pro Max
iPhone 8
iPhone 8 Plus
iPhone SE (2nd generation)
iPad (7th generation)
iPad (8th generation)
iPad Air (3rd generation)
iPad Air (4th generation)
iPad Pro (11-inch) (2nd generation)
iPad Pro (12.9-inch) (4th generation)
iPad Pro (9.7-inch) | | tvOS 13.4 | 11.7 | Apple TV
Apple TV 4K
Apple TV 4K (at 1080p) | -| tvOS 14.0 | 12.1 | Apple TV
Apple TV 4K
Apple TV 4K (at 1080p) | | tvOS 14.2 | 12.2 | Apple TV
Apple TV 4K
Apple TV 4K (at 1080p) | | watchOS 6.2 | 11.7 | Apple Watch Series 4 - 40mm
Apple Watch Series 4 - 44mm
Apple Watch Series 5 - 40mm
Apple Watch Series 5 - 44mm | -| watchOS 7.0 | 12.1 | Apple Watch Series 4 - 40mm
Apple Watch Series 4 - 44mm
Apple Watch Series 5 - 40mm
Apple Watch Series 5 - 44mm
Apple Watch Series 6 - 40mm
Apple Watch Series 6 - 44mm | | watchOS 7.1 | 12.2 | Apple Watch Series 4 - 40mm
Apple Watch Series 4 - 44mm
Apple Watch Series 5 - 40mm
Apple Watch Series 5 - 44mm
Apple Watch Series 6 - 40mm
Apple Watch Series 6 - 44mm | ### Android diff --git a/images/macos/provision/bootstrap-provisioner/installNewProvisioner.sh b/images/macos/provision/bootstrap-provisioner/installNewProvisioner.sh index 79a3afdf2..e588c670c 100644 --- a/images/macos/provision/bootstrap-provisioner/installNewProvisioner.sh +++ b/images/macos/provision/bootstrap-provisioner/installNewProvisioner.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail BOOTSTRAP_PATH="$1" ProvisionerPackageUri="$2" ProvisionerScriptUri="$3" diff --git a/images/macos/provision/configuration/finalize-vm.sh b/images/macos/provision/configuration/finalize-vm.sh index 90ffbbebd..3e73dc5f5 100644 --- a/images/macos/provision/configuration/finalize-vm.sh +++ b/images/macos/provision/configuration/finalize-vm.sh @@ -20,6 +20,9 @@ cp $HOME/image-generation/output/software-report/systeminfo.txt $HOME/image-gene # we have to do that here because `npm install` is run in a few different places during image-generation npm cache clean --force +# Clean yarn cache +yarn cache clean + # Clean up temporary directories rm -rf ~/utils ~/image-generation diff --git a/images/macos/provision/core/chrome.sh b/images/macos/provision/core/chrome.sh index 4a9deba1a..332a11d1b 100644 --- a/images/macos/provision/core/chrome.sh +++ b/images/macos/provision/core/chrome.sh @@ -1,7 +1,8 @@ #!/bin/bash -e -o pipefail +source ~/utils/utils.sh echo "Installing Chrome..." -brew cask install google-chrome +brew_cask_install_ignoring_sha256 "google-chrome" echo "Installing Chrome Driver" brew cask install chromedriver diff --git a/images/macos/provision/core/commonutils.sh b/images/macos/provision/core/commonutils.sh index c7e93496c..a479974cd 100644 --- a/images/macos/provision/core/commonutils.sh +++ b/images/macos/provision/core/commonutils.sh @@ -21,7 +21,6 @@ binst_common_utils=( gh p7zip ant - yamllint aria2 ) diff --git a/images/macos/provision/core/edge.sh b/images/macos/provision/core/edge.sh index da1673e0a..bd0ffeddb 100644 --- a/images/macos/provision/core/edge.sh +++ b/images/macos/provision/core/edge.sh @@ -3,11 +3,7 @@ source ~/utils/utils.sh echo "Installing Microsoft Edge..." -# Workaround to install version 85 since webdriver is broken for 86 -cd "$(brew --repo homebrew/homebrew-cask)" -git checkout 81f9d08d2b9b7557c0178621078cf59d2c5db2bc brew cask install microsoft-edge -git checkout master EDGE_INSTALLATION_PATH="/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge" EDGE_VERSION=$("$EDGE_INSTALLATION_PATH" --version | cut -d' ' -f 3) diff --git a/images/macos/provision/core/pipx-packages.sh b/images/macos/provision/core/pipx-packages.sh new file mode 100644 index 000000000..658600fad --- /dev/null +++ b/images/macos/provision/core/pipx-packages.sh @@ -0,0 +1,18 @@ +source ~/utils/utils.sh + +export PATH="$PATH:/opt/pipx_bin" + +toolset=$(get_toolset_path) +pipx_packages=$(jq -r ".pipx[] .package" $toolset) + +for package in $pipx_packages; do + python_version=$(jq -r ".pipx[] | select(.package == \"$package\") .python" $toolset) + if [ "$python_version" != "null" ]; then + python_path="$HOME/hostedtoolcache/Python/$python_version*/x64/bin/python$python_version" + echo "Install $package into python $python_path" + pipx install $package --python $python_path + else + echo "Install $package into default python" + pipx install $package + fi +done \ No newline at end of file diff --git a/images/macos/provision/core/python.sh b/images/macos/provision/core/python.sh index b4c641477..629220535 100755 --- a/images/macos/provision/core/python.sh +++ b/images/macos/provision/core/python.sh @@ -17,3 +17,13 @@ echo "Brew Installing Python 2" brew tap-new --no-git local/python2 FORMULA_PATH=$(brew extract python@2 local/python2 | grep "Homebrew/Library/Taps") brew install $FORMULA_PATH + +echo "Installing pipx" +export PIPX_BIN_DIR=/usr/local/opt/pipx_bin +export PIPX_HOME=/usr/local/opt/pipx + +brew install pipx + +echo "export PIPX_BIN_DIR=${PIPX_BIN_DIR}" >> "${HOME}/.bashrc" +echo "export PIPX_HOME=${PIPX_HOME}" >> "${HOME}/.bashrc" +echo 'export PATH="$PIPX_BIN_DIR:$PATH"' >> "${HOME}/.bashrc" diff --git a/images/macos/provision/core/xamarin.sh b/images/macos/provision/core/xamarin.sh index 86dc1a0fe..18eef3219 100755 --- a/images/macos/provision/core/xamarin.sh +++ b/images/macos/provision/core/xamarin.sh @@ -9,6 +9,7 @@ XAMARIN_MAC_VERSIONS=($(get_toolset_value '.xamarin."mac-versions" | reverse | . XAMARIN_ANDROID_VERSIONS=($(get_toolset_value '.xamarin."android-versions" | reverse | .[]')) LATEST_SDK_SYMLINK=$(get_toolset_value '.xamarin.bundles[0].symlink') CURRENT_SDK_SYMLINK=$(get_toolset_value '.xamarin."bundle-default"') +DEFAULT_XCODE_VERSION=$(get_default_xcode_from_toolset) if [ "$CURRENT_SDK_SYMLINK" == "latest" ]; then CURRENT_SDK_SYMLINK=$LATEST_SDK_SYMLINK @@ -82,3 +83,8 @@ popd echo "Clean up packages..." sudo rm -rf "$TMPMOUNT" + +# Fix Xamarin issue with Xcode symlink: https://github.com/xamarin/xamarin-macios/issues/9960 +PREFERENCES_XAMARIN_DIR="${HOME}/Library/Preferences/Xamarin" +mkdir -p $PREFERENCES_XAMARIN_DIR +/usr/libexec/PlistBuddy -c "add :AppleSdkRoot string /Applications/Xcode_${DEFAULT_XCODE_VERSION}.app" $PREFERENCES_XAMARIN_DIR/Settings.plist diff --git a/images/macos/provision/utils/utils.sh b/images/macos/provision/utils/utils.sh index d2c393177..c43f4efee 100755 --- a/images/macos/provision/utils/utils.sh +++ b/images/macos/provision/utils/utils.sh @@ -107,4 +107,17 @@ get_default_xcode_from_toolset() { verlte() { sortedVersion=$(echo -e "$1\n$2" | sort -V | head -n1) [ "$1" = "$sortedVersion" ] +} + +brew_cask_install_ignoring_sha256() { + local TOOL_NAME=$1 + + CASK_DIR="$(brew --repo homebrew/cask)/Casks" + chmod a+w "$CASK_DIR/$TOOL_NAME.rb" + SHA=$(grep "sha256" "$CASK_DIR/$TOOL_NAME.rb" | awk '{print $2}') + sed -i '' "s/$SHA/:no_check/" "$CASK_DIR/$TOOL_NAME.rb" + brew cask install $TOOL_NAME + pushd $CASK_DIR + git checkout HEAD -- "$TOOL_NAME.rb" + popd } \ No newline at end of file diff --git a/images/macos/software-report/SoftwareReport.Common.psm1 b/images/macos/software-report/SoftwareReport.Common.psm1 index c3fe75171..7616b81b7 100644 --- a/images/macos/software-report/SoftwareReport.Common.psm1 +++ b/images/macos/software-report/SoftwareReport.Common.psm1 @@ -96,6 +96,11 @@ function Get-PipVersion { return "${versionPart1} ${versionPart2} ${versionPart3}" } +function Get-PipxVersion { + $pipxVersion = Run-Command "pipx --version" -SuppressStderr + return "Pipx $pipxVersion" +} + function Get-NVMNodeVersionList { $nvmPath = Join-Path $env:HOME ".nvm" "nvm.sh" $nvmInitCommand = ". ${nvmPath} > /dev/null 2>&1 || true" diff --git a/images/macos/software-report/SoftwareReport.Generator.ps1 b/images/macos/software-report/SoftwareReport.Generator.ps1 index 9bcef14ce..4eab767c4 100644 --- a/images/macos/software-report/SoftwareReport.Generator.ps1 +++ b/images/macos/software-report/SoftwareReport.Generator.ps1 @@ -83,6 +83,7 @@ $npmVersion = Run-Command "npm --version" $yarnVersion = Run-Command "yarn --version" $nugetVersion = Run-Command "nuget help" | Select-Object -First 1 | Take-Part -Part 2 $pip3Version = Get-PipVersion -Version 3 +$pipxVersion = Get-PipxVersion $condaVersion = Invoke-Expression "conda --version" $rubyGemsVersion = Run-Command "gem --version" $composerVersion = Run-Command "composer --version" | Take-Part -Part 2 @@ -99,6 +100,7 @@ if ($os.IsLessThanBigSur) { $markdown += New-MDList -Style Unordered -Lines @( "Pip ${pip3Version}", + $pipxVersion, $bundlerVersion, "Carthage ${carthageVersion}", "CocoaPods ${cocoaPodsVersion}", diff --git a/images/macos/templates/macOS-10.13.json b/images/macos/templates/macOS-10.13.json index 35c7ef3c7..9ebd56e6d 100644 --- a/images/macos/templates/macOS-10.13.json +++ b/images/macos/templates/macOS-10.13.json @@ -181,7 +181,8 @@ "./provision/core/edge.sh", "./provision/core/firefox.sh", "./provision/core/toolcache-high-sierra.sh", - "./provision/core/pypy.sh" + "./provision/core/pypy.sh", + "./provision/core/pipx-packages.sh" ] }, { diff --git a/images/macos/templates/macOS-10.14.json b/images/macos/templates/macOS-10.14.json index 2687302e0..c03cc72aa 100644 --- a/images/macos/templates/macOS-10.14.json +++ b/images/macos/templates/macOS-10.14.json @@ -186,7 +186,8 @@ "./provision/core/miniconda.sh", "./provision/core/xcode-postbuild.sh", "./provision/core/toolcache.sh", - "./provision/core/pypy.sh" + "./provision/core/pypy.sh", + "./provision/core/pipx-packages.sh" ], "environment_vars": [ "GITHUB_FEED_TOKEN={{user `github_feed_token`}}" diff --git a/images/macos/templates/macOS-10.15.json b/images/macos/templates/macOS-10.15.json index a65bd72ac..cf49ef571 100644 --- a/images/macos/templates/macOS-10.15.json +++ b/images/macos/templates/macOS-10.15.json @@ -185,7 +185,8 @@ "./provision/core/firefox.sh", "./provision/core/xcode-postbuild.sh", "./provision/core/toolcache.sh", - "./provision/core/pypy.sh" + "./provision/core/pypy.sh", + "./provision/core/pipx-packages.sh" ], "environment_vars": [ "GITHUB_FEED_TOKEN={{user `github_feed_token`}}" diff --git a/images/macos/templates/macOS-11.0.json b/images/macos/templates/macOS-11.0.json index 27f39bfb0..585a93542 100644 --- a/images/macos/templates/macOS-11.0.json +++ b/images/macos/templates/macOS-11.0.json @@ -181,7 +181,8 @@ "./provision/core/chrome.sh", "./provision/core/edge.sh", "./provision/core/firefox.sh", - "./provision/core/toolcache.sh" + "./provision/core/toolcache.sh", + "./provision/core/pipx-packages.sh" ], "environment_vars": [ "GITHUB_FEED_TOKEN={{user `github_feed_token`}}" diff --git a/images/macos/tests/Linters.Tests.ps1 b/images/macos/tests/Linters.Tests.ps1 index 3b7b8e0df..31d3698cc 100644 --- a/images/macos/tests/Linters.Tests.ps1 +++ b/images/macos/tests/Linters.Tests.ps1 @@ -3,10 +3,6 @@ Import-Module "$PSScriptRoot/../helpers/Tests.Helpers.psm1" $os = Get-OSVersion Describe "Linters" { - It "yamllint" { - "yamllint --version" | Should -ReturnZeroExitCode - } - It "SwiftLint" -Skip:($os.IsHighSierra) { "swiftlint version" | Should -ReturnZeroExitCode } diff --git a/images/macos/tests/PipxPackages.Tests.ps1 b/images/macos/tests/PipxPackages.Tests.ps1 new file mode 100644 index 000000000..ed9819388 --- /dev/null +++ b/images/macos/tests/PipxPackages.Tests.ps1 @@ -0,0 +1,7 @@ +Describe "PipxPackages" { + $pipxToolset = Get-ToolsetValue "pipx" + $testCases = $pipxToolset | ForEach-Object { @{package = $_.package; cmd = $_.cmd} } + It "" -TestCases $testCases { + "$cmd" | Should -ReturnZeroExitCode + } +} \ No newline at end of file diff --git a/images/macos/tests/Python.Tests.ps1 b/images/macos/tests/Python.Tests.ps1 index 624609994..7a50a7ab8 100644 --- a/images/macos/tests/Python.Tests.ps1 +++ b/images/macos/tests/Python.Tests.ps1 @@ -32,6 +32,10 @@ Describe "Python" { "pip3 --version" | Should -ReturnZeroExitCode } + It "Pipx is available" { + "pipx --version" | Should -ReturnZeroExitCode + } + It "Pip 3 and Python 3 came from the same brew formula" { $pip3Path = Split-Path (readlink (which pip3)) $python3Path = Split-Path (readlink (which python3)) diff --git a/images/macos/toolsets/toolset-10.13.json b/images/macos/toolsets/toolset-10.13.json index 6c12251b8..3bf629685 100644 --- a/images/macos/toolsets/toolset-10.13.json +++ b/images/macos/toolsets/toolset-10.13.json @@ -194,5 +194,11 @@ "3.6" ] } + ], + "pipx": [ + { + "package": "yamllint", + "cmd": "yamllint --version" + } ] } \ No newline at end of file diff --git a/images/macos/toolsets/toolset-10.14.json b/images/macos/toolsets/toolset-10.14.json index 2abe98036..aa1b41019 100644 --- a/images/macos/toolsets/toolset-10.14.json +++ b/images/macos/toolsets/toolset-10.14.json @@ -254,5 +254,11 @@ "1.15.*" ] } + ], + "pipx": [ + { + "package": "yamllint", + "cmd": "yamllint --version" + } ] } \ No newline at end of file diff --git a/images/macos/toolsets/toolset-10.15.json b/images/macos/toolsets/toolset-10.15.json index fc0bf7983..e041c15e2 100644 --- a/images/macos/toolsets/toolset-10.15.json +++ b/images/macos/toolsets/toolset-10.15.json @@ -163,5 +163,11 @@ "1.15.*" ] } + ], + "pipx": [ + { + "package": "yamllint", + "cmd": "yamllint --version" + } ] } \ No newline at end of file diff --git a/images/macos/toolsets/toolset-11.0.json b/images/macos/toolsets/toolset-11.0.json index 5ff2280a2..cb14b3a0e 100644 --- a/images/macos/toolsets/toolset-11.0.json +++ b/images/macos/toolsets/toolset-11.0.json @@ -88,5 +88,11 @@ "1.15.*" ] } + ], + "pipx": [ + { + "package": "yamllint", + "cmd": "yamllint --version" + } ] } \ No newline at end of file diff --git a/images/win/scripts/Installers/Initialize-VM.ps1 b/images/win/scripts/Installers/Initialize-VM.ps1 index ad853bb74..027728727 100644 --- a/images/win/scripts/Installers/Initialize-VM.ps1 +++ b/images/win/scripts/Installers/Initialize-VM.ps1 @@ -128,11 +128,7 @@ if (Test-IsWin19) { } # Expand disk size of OS drive -New-Item -Path d:\ -Name cmds.txt -ItemType File -Force -Add-Content -Path d:\cmds.txt "SELECT VOLUME=C`r`nEXTEND" - -$expandResult = (diskpart /s 'd:\cmds.txt') -Write-Host $expandResult - -Write-Host "Disk sizes after expansion" -wmic logicaldisk get size,freespace,caption +$driveLetter = "C" +$size = Get-PartitionSupportedSize -DriveLetter $driveLetter +Resize-Partition -DriveLetter $driveLetter -Size $size.SizeMax +Get-Volume | Select-Object DriveLetter, SizeRemaining, Size | Sort-Object DriveLetter \ No newline at end of file diff --git a/images/win/windows2019.json b/images/win/windows2019.json index 6609b5b38..6b9f3a868 100644 --- a/images/win/windows2019.json +++ b/images/win/windows2019.json @@ -13,7 +13,7 @@ "virtual_network_resource_group_name": "{{env `VNET_RESOURCE_GROUP`}}", "virtual_network_subnet_name": "{{env `VNET_SUBNET`}}", "private_virtual_network_with_public_ip": "{{env `PRIVATE_VIRTUAL_NETWORK_WITH_PUBLIC_IP`}}", - "vm_size": "Standard_D4_v2", + "vm_size": "Standard_DS4_v2", "run_scan_antivirus": "false", "root_folder": "C:", "toolset_json_path": "{{env `TEMP`}}\\toolset.json",