diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 89216e21..60837bc1 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -52,7 +52,8 @@ jobs: inputs: targetType: 'filePath' filePath: ./images.CI/macos/select-datastore.ps1 - arguments: -VIServer "$(vcenter-server-v2)" ` + arguments: -VMName "$(VirtualMachineName)" ` + -VIServer "$(vcenter-server-v2)" ` -VIUserName "$(vcenter-username-v2)" ` -VIPassword "$(vcenter-password-v2)" @@ -121,8 +122,8 @@ jobs: condition: always() - task: PowerShell@2 - displayName: 'Move vm to cold storage' - condition: succeededOrFailed() + displayName: 'Move vm to cold storage and clear datastore tag' + condition: always() inputs: targetType: 'filePath' filePath: ./images.CI/macos/move-vm.ps1 diff --git a/images.CI/macos/move-vm.ps1 b/images.CI/macos/move-vm.ps1 index 17e6faea..37e83906 100644 --- a/images.CI/macos/move-vm.ps1 +++ b/images.CI/macos/move-vm.ps1 @@ -48,12 +48,30 @@ Import-Module $PSScriptRoot\helpers.psm1 -DisableNameChecking # Connection to a vCenter Server system Connect-VCServer -try -{ - Get-VM $VMName | Move-VM -Datastore $TargetDataStore -ErrorAction Stop - Write-Host "VM has been moved successfully to target datastore '$TargetDataStore'" +# Clear previously assigned tag with VM Name +try { + Remove-Tag $VMName -Confirm:$false +} catch { + Write-Host "Tag with $VMName doesn't exist" } -catch -{ + +$vm = Get-VM $VMName + +if ($env:AGENT_JOBSTATUS -eq 'Failed') { + try { + if($vm.PowerState -ne "PoweredOff") { + Stop-VM -VM $vm -Confirm:$false -ErrorAction Stop + } + Set-VM -VM $vm -Name "${VMName}_failed" -Confirm:$false -ErrorAction Stop + Write-Host "VM has been successfully powered off and renamed to [${VMName}_failed]" + } catch { + Write-Host "##vso[task.LogIssue type=error;]Failed to power off and rename VM '$VMName'" + } +} + +try { + Move-VM -Vm $vm -Datastore $TargetDataStore -ErrorAction Stop + Write-Host "VM has been moved successfully to target datastore '$TargetDataStore'" +} catch { Write-Host "##vso[task.LogIssue type=error;]Failed to move VM '$VMName' to target datastore '$TargetDataStore'" } \ No newline at end of file diff --git a/images.CI/macos/select-datastore.ps1 b/images.CI/macos/select-datastore.ps1 index d6477700..2884fdf4 100644 --- a/images.CI/macos/select-datastore.ps1 +++ b/images.CI/macos/select-datastore.ps1 @@ -20,6 +20,10 @@ vCenter password (Example "12345678") [CmdletBinding()] param( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$VMName, + [Parameter(Mandatory)] [ValidateNotNullOrEmpty()] [string]$VIServer, @@ -30,35 +34,72 @@ param( [Parameter(Mandatory)] [ValidateNotNullOrEmpty()] - [string]$VIPassword + [string]$VIPassword, + + [string]$TagCategory = "Busy" ) # Import helpers module Import-Module $PSScriptRoot\helpers.psm1 -DisableNameChecking +function Select-DataStore { + param ( + [string]$VMName, + [string]$TagCategory, + [string]$TemplateDatastore = "ds-local-Datastore-*", + [int]$ThresholdInGb = 400, + [int]$VMCount = 2, + [int]$Retries = 5 + ) + + # 1. Name starts with ds-local-Datastore + # 2. FreespaceGB > 400 Gb + # 3. VM count on a datastore < 2 + + Write-Host "Start Datastore selection process..." + $allDatastores = Get-Datastore -Name $templateDatastore | Where-Object { $_.State -eq "Available" } + $buildDatastore = $allDatastores | Where-Object { $_.FreeSpaceGB -ge $thresholdInGb } | Where-Object { + $vmOnDatastore = @((Get-ChildItem -Path $_.DatastoreBrowserPath).Name -notmatch "^\.").Count + $vmOnDatastore -lt $vmCount + } | Select-Object -ExpandProperty Name -First 1 + + $tag = Get-Tag -Category $TagCategory -Name $VMName -ErrorAction Ignore + if (-not $tag) + { + $tag = New-Tag -Name $VMName -Category $TagCategory + } + + New-TagAssignment -Tag $tag -Entity $buildDatastore | Out-Null + + # Wait for 60 seconds to check if any other tags are assigned to the same datastore + Start-Sleep -Seconds 60 + # Take only first 2 tags, all the others will go to the next round + $tagAssignments = (Get-TagAssignment -Entity $buildDatastore).Tag.Name | Select-Object -First 2 + $isAllow = $tagAssignments -contains $VMName + + if ($isAllow) + { + Write-Host "Datastore selected successfully" + Write-Host "##vso[task.setvariable variable=buildDatastore;issecret=true]$buildDatastore" + return + } + + # Remove the tag if datastore wasn't selected + Remove-Tag $tag -Confirm:$false + + $retries-- + if ($retries -le 0) + { + Write-Host "##vso[task.LogIssue type=error;]No datastores found for the condition" + exit 1 + } + + Write-Host "Datastore select failed, $retries left" + Select-DataStore -VMName $VMName -TagCategory $TagCategory -Retries $retries +} + # Connection to a vCenter Server system Connect-VCServer # Get a target datastore for current deployment -# 1. Name starts with ds-local-Datastore -# 2. FreespaceGB > 400 Gb -# 3. VM count on a datastore < 2 -$templateDatastore = "ds-local-Datastore-*" -$thresholdInGb = 400 -$vmCount = 2 -$allDatastores = Get-Datastore -Name $templateDatastore | Where-Object { $_.State -eq "Available" } -$buildDatastore = $allDatastores | Where-Object { $_.FreeSpaceGB -ge $thresholdInGb } | Where-Object { - $vmOnDatastore = @((Get-ChildItem -Path $_.DatastoreBrowserPath).Name -notmatch "^\.").Count - $vmOnDatastore -lt $vmCount - } | Select-Object -ExpandProperty Name -First 1 - -if ($buildDatastore) -{ - Write-Host "Datastore selected successfully" - Write-Host "##vso[task.setvariable variable=buildDatastore;issecret=true]$buildDatastore" -} -else -{ - Write-Host "##vso[task.LogIssue type=error;]No datastores found for the condition" - exit 1 -} \ No newline at end of file +Select-DataStore -VMName $VMName -TagCategory $TagCategory diff --git a/images/linux/scripts/installers/homebrew.sh b/images/linux/scripts/installers/homebrew.sh index 865923f2..dd9e8f9e 100644 --- a/images/linux/scripts/installers/homebrew.sh +++ b/images/linux/scripts/installers/homebrew.sh @@ -12,9 +12,6 @@ source $HELPER_SCRIPTS/etc-environment.sh /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" eval $(/home/linuxbrew/.linuxbrew/bin/brew shellenv) -# Make brew files and directories writable by any user -sudo chmod -R o+w $HOMEBREW_PREFIX - # Update /etc/environemnt ## Put HOMEBREW_* variables brew shellenv|grep 'export HOMEBREW'|sed -E 's/^export (.*);$/\1/' | sudo tee -a /etc/environment diff --git a/images/macos/provision/configuration/add-network-interface-detection.sh b/images/macos/provision/configuration/add-network-interface-detection.sh index e939f04b..ca9192bf 100755 --- a/images/macos/provision/configuration/add-network-interface-detection.sh +++ b/images/macos/provision/configuration/add-network-interface-detection.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail # This script was taken from https://github.com/timsutton/osx-vm-templates/blob/master/scripts/add-network-interface-detection.sh # Distributed by MIT license, license can be found at the bottom of this script diff --git a/images/macos/provision/configuration/autologin.sh b/images/macos/provision/configuration/autologin.sh index caa29a6b..02f2f06d 100755 --- a/images/macos/provision/configuration/autologin.sh +++ b/images/macos/provision/configuration/autologin.sh @@ -1,3 +1,5 @@ +#!/bin/bash -e -o pipefail + # This script was taken from https://github.com/timsutton/osx-vm-templates/blob/master/scripts/autologin.sh # Distributed by MIT license, license can be found at the bottom of this script diff --git a/images/macos/provision/configuration/configure-hostname.sh b/images/macos/provision/configuration/configure-hostname.sh index 477e0980..e0de0bf8 100644 --- a/images/macos/provision/configuration/configure-hostname.sh +++ b/images/macos/provision/configuration/configure-hostname.sh @@ -1,8 +1,8 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail # Add script for changing hostname to run on startup to prevent duplicate hostnames across the environment. Hostname and Computername should contain .local in name to avoid name resolution issues tee -a /usr/local/bin/change_hostname.sh > /dev/null <<\EOF -#!/bin/bash +#!/bin/bash -e -o pipefail name="Mac-$(python -c 'from time import time; print int(round(time() * 1000))')" scutil --set HostName "${name}.local" diff --git a/images/macos/provision/configuration/configure-machine.sh b/images/macos/provision/configuration/configure-machine.sh index dbfae673..df991c21 100644 --- a/images/macos/provision/configuration/configure-machine.sh +++ b/images/macos/provision/configuration/configure-machine.sh @@ -1,12 +1,4 @@ -#!/bin/bash - -# Enable firewall. SSH and VNC opened. Can't did it at bootstrap step, so placed it here -defaults write /Library/Preferences/com.apple.alf globalstate -int 1 - -# Setting correct time zone -echo "Configuring system time to GMT..." -rm -f /etc/localtime -ln -sf /usr/share/zoneinfo/UTC /etc/localtime +#!/bin/bash -e -o pipefail # https://developer.apple.com/documentation/webkit/testing_with_webdriver_in_safari # Safari’s executable is located at /usr/bin/safaridriver diff --git a/images/macos/provision/configuration/configure-ssh.sh b/images/macos/provision/configuration/configure-ssh.sh index f85e9efa..73095f55 100755 --- a/images/macos/provision/configuration/configure-ssh.sh +++ b/images/macos/provision/configuration/configure-ssh.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail [[ ! -d ~/.ssh ]] && mkdir ~/.ssh 2>/dev/null chmod 777 ~/.ssh diff --git a/images/macos/provision/configuration/disable-auto-updates.sh b/images/macos/provision/configuration/disable-auto-updates.sh index 3a35f5be..050f7d9a 100755 --- a/images/macos/provision/configuration/disable-auto-updates.sh +++ b/images/macos/provision/configuration/disable-auto-updates.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail # Disabling automatic updates sudo softwareupdate --schedule off diff --git a/images/macos/provision/configuration/finalize-vm.sh b/images/macos/provision/configuration/finalize-vm.sh index b01b3c23..b6975139 100644 --- a/images/macos/provision/configuration/finalize-vm.sh +++ b/images/macos/provision/configuration/finalize-vm.sh @@ -1,12 +1,16 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail + +source ~/utils/utils.sh # Close all finder windows because they can interfere with UI tests osascript -e 'tell application "Finder" to close windows' -# Ignore available updates to prevent system pop-ups -updateName=$(softwareupdate -l | grep "Title: " | awk -F[:,] '{print $2}' | awk '{$1=$1};1') -if [ ! -z "$updateName" ]; then - sudo softwareupdate --ignore "$updateName" +if is_Less_BigSur; then + # Ignore available updates to prevent system pop-ups + updateName=$(softwareupdate -l | grep "Title: " | awk -F[:,] '{print $2}' | awk '{$1=$1};1') || true + if [ ! -z "$updateName" ]; then + sudo softwareupdate --ignore "$updateName" + fi fi # Put documentation to $HOME root diff --git a/images/macos/provision/configuration/max-files.sh b/images/macos/provision/configuration/max-files.sh index a6e41f12..cd1df451 100755 --- a/images/macos/provision/configuration/max-files.sh +++ b/images/macos/provision/configuration/max-files.sh @@ -1,6 +1,4 @@ -#!/bin/bash - -set -e +#!/bin/bash -e -o pipefail Launch_Daemons="/Library/LaunchDaemons" diff --git a/images/macos/provision/configuration/ntpconf.sh b/images/macos/provision/configuration/ntpconf.sh index 09916cc3..b4f68128 100755 --- a/images/macos/provision/configuration/ntpconf.sh +++ b/images/macos/provision/configuration/ntpconf.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail echo Additional NTP servers adding into /etc/ntp.conf file... cat > /etc/ntp.conf << EOF diff --git a/images/macos/provision/configuration/preimagedata.sh b/images/macos/provision/configuration/preimagedata.sh index c269a149..8060acb7 100644 --- a/images/macos/provision/configuration/preimagedata.sh +++ b/images/macos/provision/configuration/preimagedata.sh @@ -1,4 +1,5 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail + source ~/utils/utils.sh imagedata_file="$HOME/imagedata.json" diff --git a/images/macos/provision/configuration/screensaver-off.sh b/images/macos/provision/configuration/screensaver-off.sh index c154f8d8..6844792e 100755 --- a/images/macos/provision/configuration/screensaver-off.sh +++ b/images/macos/provision/configuration/screensaver-off.sh @@ -1,4 +1,5 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail + # set screensaver idleTime to 0, to prevent turning screensaver on macUUID=`ioreg -rd1 -c IOPlatformExpertDevice | grep -i "UUID" | cut -c27-62` diff --git a/images/macos/provision/configuration/shell-change.sh b/images/macos/provision/configuration/shell-change.sh index a84216cb..5fb12a03 100644 --- a/images/macos/provision/configuration/shell-change.sh +++ b/images/macos/provision/configuration/shell-change.sh @@ -1,3 +1,5 @@ +#!/bin/bash -e -o pipefail + echo "Changing shell to bash" sudo chsh -s /bin/bash $USERNAME sudo chsh -s /bin/bash root \ No newline at end of file diff --git a/images/macos/provision/core/android-toolsets.sh b/images/macos/provision/core/android-toolsets.sh index 093a133c..2f6139c8 100755 --- a/images/macos/provision/core/android-toolsets.sh +++ b/images/macos/provision/core/android-toolsets.sh @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash -e -o pipefail source ~/utils/utils.sh function filter_components_by_version { diff --git a/images/macos/provision/core/audiodevice.sh b/images/macos/provision/core/audiodevice.sh index 4a151e3f..ec7d71d9 100644 --- a/images/macos/provision/core/audiodevice.sh +++ b/images/macos/provision/core/audiodevice.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail echo "install soundflower" brew cask install soundflower diff --git a/images/macos/provision/core/aws.sh b/images/macos/provision/core/aws.sh index 4d98d512..02009449 100644 --- a/images/macos/provision/core/aws.sh +++ b/images/macos/provision/core/aws.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail echo Installing aws... curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg" diff --git a/images/macos/provision/core/azcopy.sh b/images/macos/provision/core/azcopy.sh index 56b848e9..5098c878 100755 --- a/images/macos/provision/core/azcopy.sh +++ b/images/macos/provision/core/azcopy.sh @@ -1,5 +1,4 @@ -#!/bin/bash -set -e +#!/bin/bash -e -o pipefail AZCOPY_DOWNLOAD_URL="https://aka.ms/downloadazcopy-v10-mac" diff --git a/images/macos/provision/core/build-xcode-symlinks.sh b/images/macos/provision/core/build-xcode-symlinks.sh index 0cee8df8..60070938 100644 --- a/images/macos/provision/core/build-xcode-symlinks.sh +++ b/images/macos/provision/core/build-xcode-symlinks.sh @@ -1,4 +1,4 @@ -set -e +#!/bin/bash -e -o pipefail source ~/utils/utils.sh diff --git a/images/macos/provision/core/chrome.sh b/images/macos/provision/core/chrome.sh index 5f8b4bdd..332a11d1 100644 --- a/images/macos/provision/core/chrome.sh +++ b/images/macos/provision/core/chrome.sh @@ -1,3 +1,4 @@ +#!/bin/bash -e -o pipefail source ~/utils/utils.sh echo "Installing Chrome..." diff --git a/images/macos/provision/core/cocoapods.sh b/images/macos/provision/core/cocoapods.sh index edf2a85f..d692bf3d 100755 --- a/images/macos/provision/core/cocoapods.sh +++ b/images/macos/provision/core/cocoapods.sh @@ -1,4 +1,5 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail + echo "Installing Cocoapods..." # Setup the Cocoapods master repo diff --git a/images/macos/provision/core/commonutils.sh b/images/macos/provision/core/commonutils.sh index 5a98f004..c7e93496 100644 --- a/images/macos/provision/core/commonutils.sh +++ b/images/macos/provision/core/commonutils.sh @@ -1,5 +1,4 @@ -#!/bin/sh -set -e +#!/bin/bash -e -o pipefail source ~/utils/utils.sh diff --git a/images/macos/provision/core/dotnet.sh b/images/macos/provision/core/dotnet.sh index 78056488..36df3be9 100755 --- a/images/macos/provision/core/dotnet.sh +++ b/images/macos/provision/core/dotnet.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail ########################################################################### # The main idea of this script is to automate dotnet installs @@ -23,7 +23,7 @@ echo "Parsing dotnet SDK (except rc and preview versions) from .json..." if is_BigSur; then DOTNET_CHANNELS=( 'https://raw.githubusercontent.com/dotnet/core/master/release-notes/2.1/releases.json' - 'https://raw.githubusercontent.com/dotnet/core/master/release-notes/3.1/releases.json' + 'https://raw.githubusercontent.com/dotnet/core/master/release-notes/3.1/releases.json' ) elif is_Less_Catalina; then DOTNET_CHANNELS=( @@ -33,7 +33,7 @@ else DOTNET_CHANNELS=( 'https://raw.githubusercontent.com/dotnet/core/master/release-notes/2.1/releases.json' 'https://raw.githubusercontent.com/dotnet/core/master/release-notes/3.0/releases.json' - 'https://raw.githubusercontent.com/dotnet/core/master/release-notes/3.1/releases.json' + 'https://raw.githubusercontent.com/dotnet/core/master/release-notes/3.1/releases.json' ) fi diff --git a/images/macos/provision/core/edge.sh b/images/macos/provision/core/edge.sh index bc19a566..bd0ffedd 100644 --- a/images/macos/provision/core/edge.sh +++ b/images/macos/provision/core/edge.sh @@ -1,11 +1,9 @@ +#!/bin/bash -e -o pipefail + source ~/utils/utils.sh echo "Installing Microsoft Edge..." -# Workaround to install version 85 since webdriver is broken for 86 -cd "$(brew --repo homebrew/homebrew-cask)" -git checkout 81f9d08d2b9b7557c0178621078cf59d2c5db2bc brew cask install microsoft-edge -git checkout master EDGE_INSTALLATION_PATH="/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge" EDGE_VERSION=$("$EDGE_INSTALLATION_PATH" --version | cut -d' ' -f 3) diff --git a/images/macos/provision/core/firefox.sh b/images/macos/provision/core/firefox.sh index f6a90d61..b9ee1c5e 100644 --- a/images/macos/provision/core/firefox.sh +++ b/images/macos/provision/core/firefox.sh @@ -1,3 +1,5 @@ +#!/bin/bash -e -o pipefail + echo "Installing Firefox..." brew cask install firefox diff --git a/images/macos/provision/core/gcc.sh b/images/macos/provision/core/gcc.sh index febc0760..fefe20fd 100644 --- a/images/macos/provision/core/gcc.sh +++ b/images/macos/provision/core/gcc.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail echo "Installing GCC@8 using homebrew..." brew install gcc@8 diff --git a/images/macos/provision/core/git.sh b/images/macos/provision/core/git.sh index 55feb92b..d59ebf91 100644 --- a/images/macos/provision/core/git.sh +++ b/images/macos/provision/core/git.sh @@ -1,3 +1,5 @@ +#!/bin/bash -e -o pipefail + echo Installing Git... brew install git diff --git a/images/macos/provision/core/haskell.sh b/images/macos/provision/core/haskell.sh index 1bc11d91..b7f35e4d 100644 --- a/images/macos/provision/core/haskell.sh +++ b/images/macos/provision/core/haskell.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh export PATH="$HOME/.ghcup/bin:$PATH" diff --git a/images/macos/provision/core/miniconda.sh b/images/macos/provision/core/miniconda.sh index b214b352..876883a6 100644 --- a/images/macos/provision/core/miniconda.sh +++ b/images/macos/provision/core/miniconda.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail MINICONDA_INSTALLER="/tmp/miniconda.sh" curl -sL https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o $MINICONDA_INSTALLER diff --git a/images/macos/provision/core/mongodb.sh b/images/macos/provision/core/mongodb.sh index 1b6a5a8d..3a4440d9 100644 --- a/images/macos/provision/core/mongodb.sh +++ b/images/macos/provision/core/mongodb.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail # MongoDB object-value database # installs last version of MongoDB Community Edition diff --git a/images/macos/provision/core/node.sh b/images/macos/provision/core/node.sh index 840b433f..59e727ce 100644 --- a/images/macos/provision/core/node.sh +++ b/images/macos/provision/core/node.sh @@ -1,3 +1,5 @@ +#!/bin/bash -e -o pipefail + source ~/utils/utils.sh node_modules=( @@ -16,7 +18,6 @@ if is_Less_Catalina; then echo Installing NPM 3.x.x... npm install -g npm@3 - npm config set prefix /usr/local # This step is required to install App Center CLI echo Installing Omelette... diff --git a/images/macos/provision/core/nvm.sh b/images/macos/provision/core/nvm.sh index 527bd4c6..c02ecf43 100755 --- a/images/macos/provision/core/nvm.sh +++ b/images/macos/provision/core/nvm.sh @@ -1,5 +1,4 @@ -#!/bin/sh - +#!/bin/bash -e -o pipefail ########################################################################### # The script installs node version manager with node versions 6,8,10 and 12 # @@ -24,11 +23,8 @@ if [ $? -eq 0 ]; then nvm alias node12 lts/erbium nvm alias node13 v13 nvm alias node14 v14 - - if is_Catalina || is_BigSur; then - # set system node as default - nvm alias default system - fi + # set system node as default + nvm alias default system else echo error fi diff --git a/images/macos/provision/core/openjdk.sh b/images/macos/provision/core/openjdk.sh index ca2c5f01..c2b22baf 100644 --- a/images/macos/provision/core/openjdk.sh +++ b/images/macos/provision/core/openjdk.sh @@ -1,6 +1,4 @@ -#!/bin/sh - -set -e +#!/bin/bash -e -o pipefail source ~/utils/utils.sh diff --git a/images/macos/provision/core/openssl.sh b/images/macos/provision/core/openssl.sh index a058673a..16a317c0 100755 --- a/images/macos/provision/core/openssl.sh +++ b/images/macos/provision/core/openssl.sh @@ -1,19 +1,16 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail source ~/utils/utils.sh -echo "Installing OpenSSL..." -export PATH="/usr/local/bin:/usr/local/sbin:~/bin:$PATH" - echo Installing OpenSSL... -/usr/local/bin/brew install openssl +brew install openssl # Install OpenSSL 1.0.2t # https://www.openssl.org/policies/releasestrat.html - Version 1.0.2 will be supported until 2019-12-31 (LTS) # To preserve backward compatibility with ruby-toolcache -/usr/local/bin/brew tap-new local/openssl -FORMULA_PATH=$(/usr/local/bin/brew extract openssl local/openssl | grep "Homebrew/Library/Taps") -/usr/local/bin/brew install $FORMULA_PATH +brew tap-new --no-git local/openssl +FORMULA_PATH=$(brew extract openssl local/openssl | grep "Homebrew/Library/Taps") +brew install $FORMULA_PATH # Set OpenSSL 1.0.2t as default ln -sf /usr/local/Cellar/openssl@1.0.2t /usr/local/Cellar/openssl @@ -25,4 +22,4 @@ ln -sf ../Cellar/openssl/1.0.2t /usr/local/opt/openssl # https://github.com/microsoft/azure-pipelines-agent/blob/master/docs/start/envosx.md mkdir -p /usr/local/lib/ ln -s /usr/local/opt/openssl@1.0.2t/lib/libcrypto.1.0.0.dylib /usr/local/lib/ -ln -s /usr/local/opt/openssl@1.0.2t/lib/libssl.1.0.0.dylib /usr/local/lib/ \ No newline at end of file +ln -s /usr/local/opt/openssl@1.0.2t/lib/libssl.1.0.0.dylib /usr/local/lib/ diff --git a/images/macos/provision/core/php.sh b/images/macos/provision/core/php.sh index 08c19f94..90111448 100644 --- a/images/macos/provision/core/php.sh +++ b/images/macos/provision/core/php.sh @@ -1,5 +1,4 @@ -#!/bin/sh -set -e +#!/bin/bash -e -o pipefail echo Installing PHP brew install php diff --git a/images/macos/provision/core/postgresql.sh b/images/macos/provision/core/postgresql.sh index f69e078a..b5d7ab61 100644 --- a/images/macos/provision/core/postgresql.sh +++ b/images/macos/provision/core/postgresql.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail #Install latest version of postgresql brew install postgres @@ -6,13 +6,21 @@ brew install postgres #Service postgresql should be started before use. brew services start postgresql -#Verify that PostgreSQL is ready for accept incoming connections. -# exit codes: -# ready - 0 -# reject - 1 -# connection timeout - 2 -# incorrect credentials or parameters - 3 -pg_isready +#Verify PostgreSQL is ready for accept incoming connections +echo "Check PostgreSQL service is running" +i=10 +COMMAND='pg_isready' +while [ $i -gt 0 ]; do + echo "Check PostgreSQL service status" + eval $COMMAND && break + ((i--)) + if [ $i == 0 ]; then + echo "PostgreSQL service not ready, all attempts exhausted" + exit 1 + fi + echo "PostgreSQL service not ready, wait 10 more sec, attempts left: $i" + sleep 10 +done #Stop postgresql brew services stop postgresql \ No newline at end of file diff --git a/images/macos/provision/core/powershell.sh b/images/macos/provision/core/powershell.sh index 5717adc7..12beeca6 100644 --- a/images/macos/provision/core/powershell.sh +++ b/images/macos/provision/core/powershell.sh @@ -1,3 +1,5 @@ +#!/bin/bash -e -o pipefail + source ~/utils/utils.sh echo Installing Azure CLI... diff --git a/images/macos/provision/core/pypy.sh b/images/macos/provision/core/pypy.sh index 5f24cd5f..d55d785a 100644 --- a/images/macos/provision/core/pypy.sh +++ b/images/macos/provision/core/pypy.sh @@ -1,10 +1,9 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail ################################################################################ ## File: pypy.sh ## Desc: Installs PyPy ################################################################################ source ~/utils/utils.sh -set -e function InstallPyPy { diff --git a/images/macos/provision/core/python.sh b/images/macos/provision/core/python.sh index 1565ced2..b4c64147 100755 --- a/images/macos/provision/core/python.sh +++ b/images/macos/provision/core/python.sh @@ -1,18 +1,19 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail + source ~/utils/utils.sh echo "Installing Python Tooling" echo "Brew Installing Python 3" # Workaround to have both 3.8 & 3.9(which required by some brew formulas) in the system, but only 3.8 is linked -/usr/local/bin/brew install python@3.8 -/usr/local/bin/brew install python@3.9 -/usr/local/bin/brew unlink python@3.9 -/usr/local/bin/brew unlink python@3.8 -/usr/local/bin/brew link python@3.8 +brew install python@3.8 +brew install python@3.9 +brew unlink python@3.9 +brew unlink python@3.8 +brew link python@3.8 echo "Brew Installing Python 2" # Create local tap with formula due to python2 formula depreciation -/usr/local/bin/brew tap-new local/python2 -FORMULA_PATH=$(/usr/local/bin/brew extract python@2 local/python2 | grep "Homebrew/Library/Taps") -/usr/local/bin/brew install $FORMULA_PATH +brew tap-new --no-git local/python2 +FORMULA_PATH=$(brew extract python@2 local/python2 | grep "Homebrew/Library/Taps") +brew install $FORMULA_PATH diff --git a/images/macos/provision/core/reboot.sh b/images/macos/provision/core/reboot.sh index a6e44435..e0a2dfb8 100644 --- a/images/macos/provision/core/reboot.sh +++ b/images/macos/provision/core/reboot.sh @@ -1,2 +1,3 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail + shutdown -r now \ No newline at end of file diff --git a/images/macos/provision/core/ruby.sh b/images/macos/provision/core/ruby.sh index 19877de3..61eb33c0 100755 --- a/images/macos/provision/core/ruby.sh +++ b/images/macos/provision/core/ruby.sh @@ -1,17 +1,6 @@ -#!/bin/sh - -set -e +#!/bin/bash -e -o pipefail source ~/utils/utils.sh echo Installing Ruby... -if is_Less_BigSur; then - # We can't install latest ruby 2.7 as a default version related with bug - # https://github.com/fastlane/fastlane/issues/15397 - /usr/local/bin/brew install ruby@2.6 - ln -sf /usr/local/opt/ruby\@2.6 /usr/local/opt/ruby -else - brew install ruby -fi - - +brew install ruby diff --git a/images/macos/provision/core/rubygem.sh b/images/macos/provision/core/rubygem.sh index 872436f8..14f4af7a 100755 --- a/images/macos/provision/core/rubygem.sh +++ b/images/macos/provision/core/rubygem.sh @@ -1,17 +1,10 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail + source ~/utils/utils.sh echo Updating RubyGems... gem update --system - -# Freeze xcodeproj 1.18.0 because version 1.19.0 contains breaking changes related to CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER flag -# Related issues: -# - https://github.com/CocoaPods/CocoaPods/issues/10153 -# - https://github.com/actions/virtual-environments/issues/1804 -# Need to revisit when Cocoapods 1.10.0 is released and added to VM -gem install xcodeproj -v 1.18.0 - echo Installing xcode-install utility... gem install xcode-install --force diff --git a/images/macos/provision/core/rust.sh b/images/macos/provision/core/rust.sh index 5ef7675c..de82c7b4 100644 --- a/images/macos/provision/core/rust.sh +++ b/images/macos/provision/core/rust.sh @@ -1,6 +1,4 @@ -#!/bin/sh - -set -e +#!/bin/bash -e -o pipefail echo Installing Rustup... brew install rustup-init diff --git a/images/macos/provision/core/stack.sh b/images/macos/provision/core/stack.sh index 81b41fa8..17c5ecd9 100644 --- a/images/macos/provision/core/stack.sh +++ b/images/macos/provision/core/stack.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail echo "Get the latest Stack version..." StackRelease=$(curl -s "https://api.github.com/repos/commercialhaskell/stack/releases/latest") diff --git a/images/macos/provision/core/toolcache-high-sierra.sh b/images/macos/provision/core/toolcache-high-sierra.sh index 61a3cc8c..9a67418b 100644 --- a/images/macos/provision/core/toolcache-high-sierra.sh +++ b/images/macos/provision/core/toolcache-high-sierra.sh @@ -1,7 +1,6 @@ -#!/bin/sh - +#!/bin/bash -e -o pipefail ########################################################################### -# The script downloads macos hosted tool cache for several Python versions +# The script downloads macos hosted tool cache for several Python versions # and installs them onto the system # ########################################################################### diff --git a/images/macos/provision/core/toolcache.sh b/images/macos/provision/core/toolcache.sh index eb6dc905..dab45deb 100755 --- a/images/macos/provision/core/toolcache.sh +++ b/images/macos/provision/core/toolcache.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail # Download hosted tool cache with npm NPM_FEED="npm.pkg.github.com" diff --git a/images/macos/provision/core/vcpkg.sh b/images/macos/provision/core/vcpkg.sh index 289d7ba4..260f2c2a 100644 --- a/images/macos/provision/core/vcpkg.sh +++ b/images/macos/provision/core/vcpkg.sh @@ -1,6 +1,5 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail -set -e source ~/utils/utils.sh # Set env variable for vcpkg diff --git a/images/macos/provision/core/vsmac.sh b/images/macos/provision/core/vsmac.sh index 20a8efd4..f0c498fc 100644 --- a/images/macos/provision/core/vsmac.sh +++ b/images/macos/provision/core/vsmac.sh @@ -1,4 +1,5 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail + source ~/utils/utils.sh source ~/utils/xamarin-utils.sh @@ -22,9 +23,6 @@ echo "Moving Visual Studio to /Applications/..." pushd $TMPMOUNT tar cf - "./Visual Studio.app" | tar xf - -C /Applications/ -echo "Launching vstools..." -/Applications/Visual\ Studio.app/Contents/MacOS/vstool - popd sudo hdiutil detach "$TMPMOUNT" sudo rm -rf "$TMPMOUNT" \ No newline at end of file diff --git a/images/macos/provision/core/xamarin-android-ndk.sh b/images/macos/provision/core/xamarin-android-ndk.sh index 35e4d006..d5b23ebb 100644 --- a/images/macos/provision/core/xamarin-android-ndk.sh +++ b/images/macos/provision/core/xamarin-android-ndk.sh @@ -1,4 +1,5 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail + source ~/utils/utils.sh ANDROID_HOME=$HOME/Library/Android/sdk diff --git a/images/macos/provision/core/xamarin.sh b/images/macos/provision/core/xamarin.sh index 0f0801d7..18eef321 100755 --- a/images/macos/provision/core/xamarin.sh +++ b/images/macos/provision/core/xamarin.sh @@ -1,4 +1,5 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail + source ~/utils/utils.sh source ~/utils/xamarin-utils.sh @@ -8,6 +9,7 @@ XAMARIN_MAC_VERSIONS=($(get_toolset_value '.xamarin."mac-versions" | reverse | . XAMARIN_ANDROID_VERSIONS=($(get_toolset_value '.xamarin."android-versions" | reverse | .[]')) LATEST_SDK_SYMLINK=$(get_toolset_value '.xamarin.bundles[0].symlink') CURRENT_SDK_SYMLINK=$(get_toolset_value '.xamarin."bundle-default"') +DEFAULT_XCODE_VERSION=$(get_default_xcode_from_toolset) if [ "$CURRENT_SDK_SYMLINK" == "latest" ]; then CURRENT_SDK_SYMLINK=$LATEST_SDK_SYMLINK @@ -63,8 +65,10 @@ createBundleLink $CURRENT_SDK_SYMLINK "Current" # Fix nuget in some mono versions because of known bugs # -# Fix Mono issue with default nuget: https://github.com/mono/mono/issues/17637 -installNuget "6.4.0" "5.3.1" +if is_Less_BigSur; then + # Fix Mono issue with default nuget: https://github.com/mono/mono/issues/17637 + installNuget "6.4.0" "5.3.1" +fi if is_Less_Catalina; then installNuget "4.8.1" "4.3.0" @@ -79,3 +83,8 @@ popd echo "Clean up packages..." sudo rm -rf "$TMPMOUNT" + +# Fix Xamarin issue with Xcode symlink: https://github.com/xamarin/xamarin-macios/issues/9960 +PREFERENCES_XAMARIN_DIR="${HOME}/Library/Preferences/Xamarin" +mkdir -p $PREFERENCES_XAMARIN_DIR +/usr/libexec/PlistBuddy -c "add :AppleSdkRoot string /Applications/Xcode_${DEFAULT_XCODE_VERSION}.app" $PREFERENCES_XAMARIN_DIR/Settings.plist diff --git a/images/macos/provision/core/xcode-postbuild.sh b/images/macos/provision/core/xcode-postbuild.sh index bf7ba387..6afd42c2 100644 --- a/images/macos/provision/core/xcode-postbuild.sh +++ b/images/macos/provision/core/xcode-postbuild.sh @@ -1,11 +1,11 @@ -#!/bin/bash +#!/bin/bash -e -o pipefail source ~/utils/utils.sh XCODE_LIST=($(get_xcode_list_from_toolset)) DEFAULT_XCODE_VERSION=$(get_default_xcode_from_toolset) # https://github.com/microsoft/appcenter/issues/847 -# Assets.xcassets : error : CoreData: error: (6922) I/O error for database +# Assets.xcassets : error : CoreData: error: (6922) I/O error for database # at $HOME/Library/Developer/Xcode/UserData/IB Support/Simulator Devices/{GUID} echo "Erase a device's contents and settings:" for XCODE_VERSION in "${XCODE_LIST[@]}" @@ -15,6 +15,11 @@ do #add sleep to let CoreSimulatorService to exit sleep 3 + # Version 12.2_beta installed into 12.2 directory and 12.1_GM_seed in 12.1 + pattern="[0-9]{1,2}.*_" + if [[ $XCODE_VERSION =~ $pattern ]] ; then + XCODE_VERSION=$(echo $XCODE_VERSION | cut -d"_" -f 1) + fi # Select xcode version by default sudo xcode-select -s "/Applications/Xcode_${XCODE_VERSION}.app/Contents/Developer" diff --git a/images/macos/provision/core/xcode-sims.sh b/images/macos/provision/core/xcode-sims.sh index fb2af5d4..8dff14a5 100755 --- a/images/macos/provision/core/xcode-sims.sh +++ b/images/macos/provision/core/xcode-sims.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail source ~/utils/utils.sh diff --git a/images/macos/provision/core/xcode-tools.sh b/images/macos/provision/core/xcode-tools.sh index 643dc293..773b07a9 100755 --- a/images/macos/provision/core/xcode-tools.sh +++ b/images/macos/provision/core/xcode-tools.sh @@ -1,10 +1,8 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail # The script currently requires 2 external variables to be set: XCODE_INSTALL_USER # and XCODE_INSTALL_PASSWORD, in order to access the Apple Developer Center -set -e - source ~/utils/utils.sh source ~/utils/xcode-utils.sh diff --git a/images/macos/provision/utils/utils.sh b/images/macos/provision/utils/utils.sh index d2f1bd99..c43f4efe 100755 --- a/images/macos/provision/utils/utils.sh +++ b/images/macos/provision/utils/utils.sh @@ -1,3 +1,5 @@ +#!/bin/bash -e -o pipefail + download_with_retries() { # Due to restrictions of bash functions, positional arguments are used here. # In case if you using latest argument NAME, you should also set value to all previous parameters. @@ -5,22 +7,31 @@ download_with_retries() { local URL="$1" local DEST="${2:-.}" local NAME="${3:-${URL##*/}}" + local COMPRESSED="$4" - echo "Downloading $URL..." - wget $URL --output-document="$DEST/$NAME" \ - --tries=30 \ - --wait 30 \ - --retry-connrefused \ - --retry-on-host-error \ - --retry-on-http-error=404,429,500,502,503 \ - --no-verbose - - if [ $? != 0 ]; then - echo "Could not download $URL; Exiting build!" - exit 1 + if [[ $COMPRESSED == "compressed" ]]; then + COMMAND="curl $URL -4 -sL --compressed -o '$DEST/$NAME'" + else + COMMAND="curl $URL -4 -sL -o '$DEST/$NAME'" fi - return 0 + echo "Downloading $URL..." + retries=20 + interval=30 + while [ $retries -gt 0 ]; do + ((retries--)) + eval $COMMAND + if [ $? != 0 ]; then + echo "Unable to download $URL, next attempt in $interval sec, $retries attempts left" + sleep $interval + else + echo "$URL was downloaded successfully to $DEST/$NAME" + return 0 + fi + done + + echo "Could not download $URL" + return 1 } is_BigSur() { diff --git a/images/macos/provision/utils/xamarin-utils.sh b/images/macos/provision/utils/xamarin-utils.sh index dd6d7ae9..1017cec2 100644 --- a/images/macos/provision/utils/xamarin-utils.sh +++ b/images/macos/provision/utils/xamarin-utils.sh @@ -1,11 +1,11 @@ -#!/bin/sh +#!/bin/bash -e -o pipefail # Xamarin can clean their SDKs while updating to newer versions, # so we should be able to detect it during image generation downloadAndInstallPKG() { local PKG_URL=$1 local PKG_NAME=${PKG_URL##*/} - + download_with_retries $PKG_URL echo "Installing $PKG_NAME..." @@ -159,7 +159,7 @@ installNunitConsole() { local MONO_VERSION=$1 cat < ${TMPMOUNT}/${NUNIT3_CONSOLE_BIN} -#!/bin/sh +#!/bin/bash -e -o pipefail exec /Library/Frameworks/Mono.framework/Versions/${MONO_VERSION}/bin/mono --debug \$MONO_OPTIONS $NUNIT3_PATH/nunit3-console.exe "\$@" EOF sudo chmod +x ${TMPMOUNT}/${NUNIT3_CONSOLE_BIN} @@ -180,7 +180,7 @@ downloadNUnitConsole() { echo "Installing NUnit 3..." sudo unzip nunit3.zip -d $NUNIT3_PATH NUNIT3_CONSOLE_BIN=nunit3-console - + popd } diff --git a/images/macos/provision/utils/xcode-utils.sh b/images/macos/provision/utils/xcode-utils.sh index 7cd63b2f..cfe93439 100644 --- a/images/macos/provision/utils/xcode-utils.sh +++ b/images/macos/provision/utils/xcode-utils.sh @@ -1,3 +1,5 @@ +#!/bin/bash -e -o pipefail + createXamarinProvisionatorSymlink() { local XCODE_VERSION="$1" local FULL_VERSION=$(echo "${XCODE_VERSION}.0.0" | cut -d'.' -f 1,2,3) diff --git a/images/macos/toolsets/toolset-10.15.json b/images/macos/toolsets/toolset-10.15.json index d3c1866d..fc0bf798 100644 --- a/images/macos/toolsets/toolset-10.15.json +++ b/images/macos/toolsets/toolset-10.15.json @@ -19,7 +19,7 @@ "android-versions": [ "11.0.2.0", "10.3.1.4", "10.2.0.100", "10.1.3.7", "10.0.6.2" ], - "bundle-default": "6_12_0", + "bundle-default": "latest", "bundles": [ { "symlink": "6_12_1", diff --git a/images/macos/toolsets/toolset-11.0.json b/images/macos/toolsets/toolset-11.0.json index 18d72c23..5ff2280a 100644 --- a/images/macos/toolsets/toolset-11.0.json +++ b/images/macos/toolsets/toolset-11.0.json @@ -2,7 +2,7 @@ "xcode": { "default": "11.7", "versions": [ - "12.2_beta", "12.1_GM_seed", "11.7" + "12.2_beta", "11.7" ] }, "xamarin": { diff --git a/images/win/scripts/Installers/Initialize-VM.ps1 b/images/win/scripts/Installers/Initialize-VM.ps1 index ad853bb7..02772872 100644 --- a/images/win/scripts/Installers/Initialize-VM.ps1 +++ b/images/win/scripts/Installers/Initialize-VM.ps1 @@ -128,11 +128,7 @@ if (Test-IsWin19) { } # Expand disk size of OS drive -New-Item -Path d:\ -Name cmds.txt -ItemType File -Force -Add-Content -Path d:\cmds.txt "SELECT VOLUME=C`r`nEXTEND" - -$expandResult = (diskpart /s 'd:\cmds.txt') -Write-Host $expandResult - -Write-Host "Disk sizes after expansion" -wmic logicaldisk get size,freespace,caption +$driveLetter = "C" +$size = Get-PartitionSupportedSize -DriveLetter $driveLetter +Resize-Partition -DriveLetter $driveLetter -Size $size.SizeMax +Get-Volume | Select-Object DriveLetter, SizeRemaining, Size | Sort-Object DriveLetter \ No newline at end of file diff --git a/images/win/windows2019.json b/images/win/windows2019.json index 6609b5b3..6b9f3a86 100644 --- a/images/win/windows2019.json +++ b/images/win/windows2019.json @@ -13,7 +13,7 @@ "virtual_network_resource_group_name": "{{env `VNET_RESOURCE_GROUP`}}", "virtual_network_subnet_name": "{{env `VNET_SUBNET`}}", "private_virtual_network_with_public_ip": "{{env `PRIVATE_VIRTUAL_NETWORK_WITH_PUBLIC_IP`}}", - "vm_size": "Standard_D4_v2", + "vm_size": "Standard_DS4_v2", "run_scan_antivirus": "false", "root_folder": "C:", "toolset_json_path": "{{env `TEMP`}}\\toolset.json",