[macOS] Fail builds on errors during the image generation (#1756)

* set -e and fix all the scripts

* add source utils to finalize_vm script

* change xcode version in postbuild script

* fix for softwareupdates and for xcode version

images/macos/provision/configuration/add-network-interface-detection.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
 
 # This script was taken from https://github.com/timsutton/osx-vm-templates/blob/master/scripts/add-network-interface-detection.sh
 # Distributed by MIT license, license can be found at the bottom of this script

images/macos/provision/configuration/autologin.sh

@@ -1,3 +1,5 @@
+#!/bin/bash -e -o pipefail
+
 # This script was taken from https://github.com/timsutton/osx-vm-templates/blob/master/scripts/autologin.sh
 # Distributed by MIT license, license can be found at the bottom of this script
 

images/macos/provision/configuration/configure-hostname.sh

@@ -1,8 +1,8 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 
 # Add script for changing hostname to run on startup to prevent duplicate hostnames across the environment. Hostname and Computername should contain .local in name to avoid name resolution issues
 tee -a /usr/local/bin/change_hostname.sh > /dev/null <<\EOF
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 
 name="Mac-$(python -c 'from time import time; print int(round(time() * 1000))')"
 scutil --set HostName "${name}.local"

images/macos/provision/configuration/configure-machine.sh

@@ -1,12 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
-
-# Enable firewall. SSH and VNC opened. Can't did it at bootstrap step, so placed it here
-defaults write /Library/Preferences/com.apple.alf globalstate -int 1
-
-# Setting correct time zone
-echo "Configuring system time to GMT..."
-rm -f /etc/localtime
-ln -sf /usr/share/zoneinfo/UTC /etc/localtime
 
 # https://developer.apple.com/documentation/webkit/testing_with_webdriver_in_safari
 # Safari’s executable is located at /usr/bin/safaridriver

images/macos/provision/configuration/configure-ssh.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
 
 [[ ! -d ~/.ssh ]] && mkdir ~/.ssh 2>/dev/null
 chmod 777 ~/.ssh

images/macos/provision/configuration/disable-auto-updates.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 
 # Disabling automatic updates
 sudo softwareupdate --schedule off

images/macos/provision/configuration/finalize-vm.sh

@@ -1,12 +1,16 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
+
+source ~/utils/utils.sh
 
 # Close all finder windows because they can interfere with UI tests
 osascript -e 'tell application "Finder" to close windows'
 
-# Ignore available updates to prevent system pop-ups
+if is_Less_BigSur; then
-updateName=$(softwareupdate -l | grep "Title: " | awk -F[:,] '{print $2}' | awk '{$1=$1};1')
+    # Ignore available updates to prevent system pop-ups
-if [ ! -z "$updateName" ]; then
+    updateName=$(softwareupdate -l | grep "Title: " | awk -F[:,] '{print $2}' | awk '{$1=$1};1') || true
-    sudo softwareupdate --ignore "$updateName"
+    if [ ! -z "$updateName" ]; then
+        sudo softwareupdate --ignore "$updateName"
+    fi
 fi
 
 # Put documentation to $HOME root

images/macos/provision/configuration/max-files.sh

@@ -1,6 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
-
-set -e
 
 Launch_Daemons="/Library/LaunchDaemons"
 

images/macos/provision/configuration/ntpconf.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 
 echo Additional NTP servers adding into /etc/ntp.conf file...
 cat > /etc/ntp.conf << EOF

images/macos/provision/configuration/preimagedata.sh

@@ -1,4 +1,5 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
+
 source ~/utils/utils.sh
 
 imagedata_file="$HOME/imagedata.json"

images/macos/provision/configuration/screensaver-off.sh

@@ -1,4 +1,5 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
+
 # set screensaver idleTime to 0, to prevent turning screensaver on
 macUUID=`ioreg -rd1 -c IOPlatformExpertDevice | grep -i "UUID" | cut -c27-62`
 

images/macos/provision/configuration/shell-change.sh

@@ -1,3 +1,5 @@
+#!/bin/bash -e -o pipefail
+
 echo "Changing shell to bash"
 sudo chsh -s /bin/bash $USERNAME
 sudo chsh -s /bin/bash root

images/macos/provision/core/android-toolsets.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/bin/bash -e -o pipefail
 source ~/utils/utils.sh
 
 function filter_components_by_version {

images/macos/provision/core/audiodevice.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 
 echo "install soundflower"
 brew cask install soundflower

images/macos/provision/core/aws.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 
 echo Installing aws...
 curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg"

images/macos/provision/core/azcopy.sh

@@ -1,5 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
-set -e
 
 AZCOPY_DOWNLOAD_URL="https://aka.ms/downloadazcopy-v10-mac"
 

images/macos/provision/core/build-xcode-symlinks.sh

@@ -1,4 +1,4 @@
-set -e
+#!/bin/bash -e -o pipefail
 
 source ~/utils/utils.sh
 

images/macos/provision/core/chrome.sh

@@ -1,3 +1,5 @@
+#!/bin/bash -e -o pipefail
+
 echo "Installing Chrome..."
 brew cask install google-chrome
 

images/macos/provision/core/cocoapods.sh

@@ -1,4 +1,5 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
+
 echo "Installing Cocoapods..."
 
 # Setup the Cocoapods master repo

images/macos/provision/core/commonutils.sh

@@ -1,5 +1,4 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
-set -e
 
 source ~/utils/utils.sh
 

images/macos/provision/core/dotnet.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
 
 ###########################################################################
 # The main idea of this script is to automate dotnet installs
@@ -23,7 +23,7 @@ echo "Parsing dotnet SDK (except rc and preview versions) from .json..."
 if is_BigSur; then
     DOTNET_CHANNELS=(
     'https://raw.githubusercontent.com/dotnet/core/master/release-notes/2.1/releases.json'
-    'https://raw.githubusercontent.com/dotnet/core/master/release-notes/3.1/releases.json' 
+    'https://raw.githubusercontent.com/dotnet/core/master/release-notes/3.1/releases.json'
     )
 elif is_Less_Catalina; then
     DOTNET_CHANNELS=(
@@ -33,7 +33,7 @@ else
     DOTNET_CHANNELS=(
     'https://raw.githubusercontent.com/dotnet/core/master/release-notes/2.1/releases.json'
     'https://raw.githubusercontent.com/dotnet/core/master/release-notes/3.0/releases.json'
-    'https://raw.githubusercontent.com/dotnet/core/master/release-notes/3.1/releases.json' 
+    'https://raw.githubusercontent.com/dotnet/core/master/release-notes/3.1/releases.json'
     )
 fi
 

images/macos/provision/core/edge.sh

@@ -1,3 +1,5 @@
+#!/bin/bash -e -o pipefail
+
 source ~/utils/utils.sh
 
 echo "Installing Microsoft Edge..."

images/macos/provision/core/firefox.sh

@@ -1,3 +1,5 @@
+#!/bin/bash -e -o pipefail
+
 echo "Installing Firefox..."
 brew cask install firefox
 

images/macos/provision/core/gcc.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 
 echo "Installing GCC@8 using homebrew..."
 brew install gcc@8

images/macos/provision/core/git.sh

@@ -1,3 +1,5 @@
+#!/bin/bash -e -o pipefail
+
 echo Installing Git...
 brew install git
 

images/macos/provision/core/haskell.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 
 curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh
 export PATH="$HOME/.ghcup/bin:$PATH"

images/macos/provision/core/miniconda.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 
 MINICONDA_INSTALLER="/tmp/miniconda.sh"
 curl -sL https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o $MINICONDA_INSTALLER

images/macos/provision/core/mongodb.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
 
 # MongoDB object-value database
 # installs last version of MongoDB Community Edition

images/macos/provision/core/node.sh

@@ -1,3 +1,5 @@
+#!/bin/bash -e -o pipefail
+
 source ~/utils/utils.sh
 
 node_modules=(
@@ -16,7 +18,6 @@ if is_Less_Catalina; then
 
   echo Installing NPM 3.x.x...
   npm install -g npm@3
-  npm config set prefix /usr/local
 
   # This step is required to install App Center CLI
   echo Installing Omelette...

images/macos/provision/core/nvm.sh

@@ -1,5 +1,4 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
-
 ###########################################################################
 # The script installs node version manager with node versions 6,8,10 and 12
 #
@@ -24,11 +23,8 @@ if [ $? -eq 0 ]; then
         nvm alias node12 lts/erbium
         nvm alias node13 v13
         nvm alias node14 v14
-
+        # set system node as default
-        if is_Catalina || is_BigSur; then
+        nvm alias default system
-                # set system node as default
-                nvm alias default system
-        fi
 else
         echo error
 fi

images/macos/provision/core/openjdk.sh

@@ -1,6 +1,4 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
-
-set -e
 
 source ~/utils/utils.sh
 

images/macos/provision/core/openssl.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
 
 source ~/utils/utils.sh
 

images/macos/provision/core/php.sh

@@ -1,5 +1,4 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
-set -e
 
 echo Installing PHP
 brew install php

images/macos/provision/core/postgresql.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 
 #Install latest version of postgresql
 brew install postgres
@@ -6,13 +6,21 @@ brew install postgres
 #Service postgresql should be started before use.
 brew services start postgresql
 
-#Verify that PostgreSQL is ready for accept incoming connections.
+#Verify PostgreSQL is ready for accept incoming connections
-# exit codes:
+echo "Check PostgreSQL service is running"
-# ready - 0
+i=10
-# reject - 1
+COMMAND='pg_isready'
-# connection timeout - 2
+while [ $i -gt 0 ]; do
-# incorrect credentials or parameters - 3
+    echo "Check PostgreSQL service status"
-pg_isready
+    eval $COMMAND && break
+    ((i--))
+    if [ $i == 0 ]; then
+        echo "PostgreSQL service not ready, all attempts exhausted"
+        exit 1
+    fi
+    echo "PostgreSQL service not ready, wait 10 more sec, attempts left: $i"
+    sleep 10
+done
 
 #Stop postgresql
 brew services stop postgresql

images/macos/provision/core/powershell.sh

@@ -1,3 +1,5 @@
+#!/bin/bash -e -o pipefail
+
 source ~/utils/utils.sh
 
 echo Installing Azure CLI...

images/macos/provision/core/pypy.sh

@@ -1,10 +1,9 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 ################################################################################
 ##  File:  pypy.sh
 ##  Desc:  Installs PyPy
 ################################################################################
 source ~/utils/utils.sh
-set -e
 
 function InstallPyPy
 {

images/macos/provision/core/python.sh

@@ -1,4 +1,5 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
+
 source ~/utils/utils.sh
 
 echo "Installing Python Tooling"

images/macos/provision/core/reboot.sh

@@ -1,2 +1,3 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
+
 shutdown -r now

images/macos/provision/core/ruby.sh

@@ -1,17 +1,6 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
-
-set -e
 
 source ~/utils/utils.sh
 
 echo Installing Ruby...
-if is_Less_BigSur; then
+brew install ruby
-    # We can't install latest ruby 2.7 as a default version related with bug
-    # https://github.com/fastlane/fastlane/issues/15397
-    /usr/local/bin/brew install ruby@2.6
-    ln -sf /usr/local/opt/ruby\@2.6 /usr/local/opt/ruby
-else
-    brew install ruby
-fi
-
-

images/macos/provision/core/rubygem.sh

@@ -1,4 +1,5 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
+
 source ~/utils/utils.sh
 
 echo Updating RubyGems...

images/macos/provision/core/rust.sh

@@ -1,6 +1,4 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
-
-set -e
 
 echo Installing Rustup...
 brew install rustup-init

images/macos/provision/core/stack.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 
 echo "Get the latest Stack version..."
 StackRelease=$(curl -s "https://api.github.com/repos/commercialhaskell/stack/releases/latest")

images/macos/provision/core/toolcache-high-sierra.sh

@@ -1,7 +1,6 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
-
 ###########################################################################
-# The script downloads macos hosted tool cache for several Python versions 
+# The script downloads macos hosted tool cache for several Python versions
 # and installs them onto the system
 #
 ###########################################################################

images/macos/provision/core/toolcache.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
 
 # Download hosted tool cache with npm
 NPM_FEED="npm.pkg.github.com"

images/macos/provision/core/vcpkg.sh

@@ -1,6 +1,5 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 
-set -e
 source ~/utils/utils.sh
 
 # Set env variable for vcpkg

images/macos/provision/core/vsmac.sh

@@ -1,4 +1,5 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
+
 source ~/utils/utils.sh
 source ~/utils/xamarin-utils.sh
 
@@ -22,9 +23,6 @@ echo "Moving Visual Studio to /Applications/..."
 pushd $TMPMOUNT
 tar cf - "./Visual Studio.app" | tar xf - -C /Applications/
 
-echo "Launching vstools..."
-/Applications/Visual\ Studio.app/Contents/MacOS/vstool
-
 popd
 sudo hdiutil detach "$TMPMOUNT"
 sudo rm -rf "$TMPMOUNT"

images/macos/provision/core/xamarin-android-ndk.sh

@@ -1,4 +1,5 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
+
 source ~/utils/utils.sh
 
 ANDROID_HOME=$HOME/Library/Android/sdk

images/macos/provision/core/xamarin.sh

@@ -1,4 +1,5 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
+
 source ~/utils/utils.sh
 source ~/utils/xamarin-utils.sh
 
@@ -63,8 +64,10 @@ createBundleLink $CURRENT_SDK_SYMLINK "Current"
 # Fix nuget in some mono versions because of known bugs
 #
 
-# Fix Mono issue with default nuget: https://github.com/mono/mono/issues/17637
+if is_Less_BigSur; then
-installNuget "6.4.0" "5.3.1"
+  # Fix Mono issue with default nuget: https://github.com/mono/mono/issues/17637
+  installNuget "6.4.0" "5.3.1"
+fi
 
 if is_Less_Catalina; then
   installNuget "4.8.1" "4.3.0"

images/macos/provision/core/xcode-postbuild.sh

@@ -1,11 +1,11 @@
-#!/bin/bash
+#!/bin/bash -e -o pipefail
 
 source ~/utils/utils.sh
 XCODE_LIST=($(get_xcode_list_from_toolset))
 DEFAULT_XCODE_VERSION=$(get_default_xcode_from_toolset)
 
 # https://github.com/microsoft/appcenter/issues/847
-# Assets.xcassets : error : CoreData: error: (6922) I/O error for database 
+# Assets.xcassets : error : CoreData: error: (6922) I/O error for database
 # at $HOME/Library/Developer/Xcode/UserData/IB Support/Simulator Devices/{GUID}
 echo "Erase a device's contents and settings:"
 for XCODE_VERSION in "${XCODE_LIST[@]}"
@@ -15,6 +15,11 @@ do
     #add sleep to let CoreSimulatorService to exit
     sleep 3
 
+    # Version 12.2_beta installed into 12.2 directory and 12.1_GM_seed in 12.1
+    pattern="[0-9]{1,2}.*_"
+    if [[ $XCODE_VERSION =~ $pattern ]] ; then
+        XCODE_VERSION=$(echo $XCODE_VERSION | cut -d"_" -f 1)
+    fi
     # Select xcode version by default
     sudo xcode-select -s "/Applications/Xcode_${XCODE_VERSION}.app/Contents/Developer"
 

images/macos/provision/core/xcode-sims.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
 
 source ~/utils/utils.sh
 

images/macos/provision/core/xcode-tools.sh

@@ -1,10 +1,8 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
 
 # The script currently requires 2 external variables to be set: XCODE_INSTALL_USER
 # and XCODE_INSTALL_PASSWORD, in order to access the Apple Developer Center
 
-set -e
-
 source ~/utils/utils.sh
 source ~/utils/xcode-utils.sh
 

images/macos/provision/utils/utils.sh

@@ -1,3 +1,5 @@
+#!/bin/bash -e -o pipefail
+
 download_with_retries() {
 # Due to restrictions of bash functions, positional arguments are used here.
 # In case if you using latest argument NAME, you should also set value to all previous parameters.

images/macos/provision/utils/xamarin-utils.sh

@@ -1,11 +1,11 @@
-#!/bin/sh
+#!/bin/bash -e -o pipefail
 
 # Xamarin can clean their SDKs while updating to newer versions,
 # so we should be able to detect it during image generation
 downloadAndInstallPKG() {
   local PKG_URL=$1
   local PKG_NAME=${PKG_URL##*/}
-  
+
   download_with_retries $PKG_URL
 
   echo "Installing $PKG_NAME..."
@@ -159,7 +159,7 @@ installNunitConsole() {
   local MONO_VERSION=$1
 
   cat <<EOF > ${TMPMOUNT}/${NUNIT3_CONSOLE_BIN}
-#!/bin/sh
+#!/bin/bash -e -o pipefail
 exec /Library/Frameworks/Mono.framework/Versions/${MONO_VERSION}/bin/mono --debug \$MONO_OPTIONS $NUNIT3_PATH/nunit3-console.exe "\$@"
 EOF
   sudo chmod +x ${TMPMOUNT}/${NUNIT3_CONSOLE_BIN}
@@ -180,7 +180,7 @@ downloadNUnitConsole() {
     echo "Installing NUnit 3..."
     sudo unzip nunit3.zip -d $NUNIT3_PATH
     NUNIT3_CONSOLE_BIN=nunit3-console
-    
+
     popd
 }
 

images/macos/provision/utils/xcode-utils.sh

@@ -1,3 +1,5 @@
+#!/bin/bash -e -o pipefail
+
 createXamarinProvisionatorSymlink() {
     local XCODE_VERSION="$1"
     local FULL_VERSION=$(echo "${XCODE_VERSION}.0.0" | cut -d'.' -f 1,2,3)