[macOS] Pester tests fail on error (#2378)

* [macOS] Fail Pester tests on error

* return it back

* [macOS] Pester tests fail on error

* some tests were moved

* appcenter cli moved

* openssl separate describe

* added new tests for java

* moved stack from haskell

* invoke tests to stack

* xamarin added tests sources

* mongo tests fix

* ndk path

* xamarin-android-ndk-tests removed

* fix nitpicks

* added new filename for XamarinNDK

* AzCopy Describe change

* added azcopy C correct.

* android additional_tools tests

* remove null elements from massive

* added new lines for verification macos 10.13

* add Android Tests for mac, not for linux

images/macos/helpers/Tests.Helpers.psm1

@@ -115,5 +115,6 @@ function Invoke-PesterTests {
     # Fail in case if no tests are run
     if (-not ($results -and ($results.FailedCount -eq 0) -and ($results.PassedCount -gt 0))) {
         $results
+        throw "Test run has failed"
     }
 }

images/macos/provision/core/azcopy.sh

@@ -14,4 +14,4 @@ chmod +x "/usr/local/bin/azcopy"
 echo "Done, cleaning up"
 rm -rf azcopy*
 
-invoke_tests "Common" "Azcopy"
+invoke_tests "Common" "AzCopy"

images/macos/provision/core/stack.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -e -o pipefail
-
+source ~/utils/invoke-tests.sh
 source ~/utils/utils.sh
 
 echo "Get the latest Stack version..."
@@ -22,3 +22,5 @@ tar -xzf $StackArchive -C $DestinationPath --strip 1
 touch $StackToolcachePath/x64.complete
 
 echo "export PATH="\$PATH":$DestinationPath" >> "$HOME/.bashrc"
+
+invoke_tests "Common" "Stack"

images/macos/provision/core/xamarin-android-ndk.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -e -o pipefail
-
+source ~/utils/invoke-tests.sh
 source ~/utils/utils.sh
 
 ANDROID_HOME=$HOME/Library/Android/sdk

images/macos/provision/utils/invoke-tests.sh

@@ -4,6 +4,7 @@ invoke_tests() {
     local TEST_FILE="$1"
     local TEST_NAME="$2"
 
+    source $HOME/.bashrc
     pwsh -Command "Import-Module '$HOME/image-generation/helpers/Tests.Helpers.psm1' -DisableNameChecking
         Invoke-PesterTests -TestFile \"$TEST_FILE\" -TestName \"$TEST_NAME\""
 }

images/macos/tests/Android.Tests.ps1

@@ -5,7 +5,6 @@ Import-Module "$PSScriptRoot/../software-report/SoftwareReport.Android.psm1"
 $os = Get-OSVersion
 
 Describe "Android" {
-    $androidNdkToolchains = @("mips64el-linux-android-4.9", "mipsel-linux-android-4.9")
     $androidSdkManagerPackages = Get-AndroidPackages
     [int]$platformMinVersion = Get-ToolsetValue "android.platform_min_version"
     [version]$buildToolsMinVersion = Get-ToolsetValue "android.build_tools_min_version"
@@ -31,6 +30,9 @@ Describe "Android" {
         (Get-ToolsetValue "android.additional-tools")
     ) | ForEach-Object { $_ }
 
+    # Remove empty strings from array to avoid possible issues
+    $androidPackages = $androidPackages | Where-Object {$_}
+
     BeforeAll {
         $ANDROID_SDK_DIR = Join-Path $env:HOME "Library" "Android" "sdk"
 
@@ -48,7 +50,6 @@ Describe "Android" {
         }
     }
 
-
     Context "Packages" {
         $testCases = $androidPackages | ForEach-Object { @{ PackageName = $_ } }
 
@@ -58,41 +59,8 @@ Describe "Android" {
         }
     }
 
-    Context "NDK toolchains" -Skip:($os.IsBigSur) {
-        $testCases = $androidNdkToolchains | ForEach-Object { @{AndroidNdkToolchain = $_} }
-
-        It "<AndroidNdkToolchain>" -TestCases $testCases {
-            param ([string] $AndroidNdkToolchain)
-
-            $toolchainPath = Join-Path $ANDROID_SDK_DIR "ndk-bundle" "toolchains" $AndroidNdkToolchain
-            $toolchainPath | Should -Exist
-        }
-    }
-
-    Context "Legacy NDK versions" -Skip:($os.IsBigSur) {
-        It "Android NDK version r18b is installed" {
-            $ndk18BundlePath = Join-Path $ANDROID_SDK_DIR "ndk" "18.1.5063045" "source.properties"
-            $rawContent = Get-Content $ndk18BundlePath -Raw
-            $rawContent | Should -BeLikeExactly "*Revision = 18.*"
-        }
-    }
-
     It "HAXM is installed" {
         $haxmPath = Join-Path $ANDROID_SDK_DIR "extras" "intel" "Hardware_Accelerated_Execution_Manager" "silent_install.sh"
         "$haxmPath -v" | Should -ReturnZeroExitCode
     }
 }
-
-Describe "Gradle" {
-    It "Gradle is installed" {
-        "gradle --version" | Should -ReturnZeroExitCode
-    }
-
-    It "Gradle is installed to /usr/local/bin" {
-        (Get-Command "gradle").Path | Should -BeExactly "/usr/local/bin/gradle"
-    }
-
-    It "Gradle is compatible with init.d plugins" {
-        "cd /tmp && gradle tasks" | Should -ReturnZeroExitCode
-    }
-}

images/macos/tests/Common.Tests.ps1

@@ -174,31 +174,6 @@ Describe "Common utilities" {
         "bundler --version" | Should -ReturnZeroExitCode
     }
 
-    It "Maven" {
-        "mvn --version" | Should -ReturnZeroExitCode
-    }
-
-    It "App Center CLI" {
-        "appcenter --version" | Should -ReturnZeroExitCode
-    }
-
-    It "Azure CLI" {
-        "az -v" | Should -ReturnZeroExitCode
-    }
-
-    Describe "AWS" {
-        It "AWS CLI" {
-            "aws --version" | Should -ReturnZeroExitCode
-        }
-        It "AWS SAM CLI" {
-            "sam --version" | Should -ReturnZeroExitCode
-        }
-
-        It "AWS Session Manager CLI" {
-            "session-manager-plugin --version" | Should -ReturnZeroExitCode
-        }
-    }
-
     Context "Nomad" -Skip:($os.IsBigSur) {
         It "Nomad CLI" {
             $result = Get-CommandResult "gem list"
@@ -209,13 +184,6 @@ Describe "Common utilities" {
             "ipa --version" | Should -ReturnZeroExitCode
         }
     }
-    Describe "Miniconda" {
-        It "Conda" {
-            Get-EnvironmentVariable "CONDA" | Should -Not -BeNullOrEmpty
-            $condaBinPath = Join-Path $env:CONDA "bin" "conda"
-            "$condaBinPath --version" | Should -ReturnZeroExitCode
-        }
-    }
 
     It "Fastlane" {
         "fastlane --version" | Should -ReturnZeroExitCode
@@ -225,29 +193,6 @@ Describe "Common utilities" {
         "xcpretty --version" | Should -ReturnZeroExitCode
     }
 
-    Describe "Mongo" {
-        It "mongodb" {
-            "mongo --version" | Should -ReturnZeroExitCode
-            "mongod --version"| Should -ReturnZeroExitCode
-        }
-    }
-
-    Describe "OpenSSL" {
-        It "OpenSSL is available" {
-            "openssl version" | Should -ReturnZeroExitCode
-        }
-
-        It "OpenSSL 1.1 path exists" {
-            $openSSLpath = "/usr/local/opt/openssl@1.1"
-            $openSSLpath | Should -Exist
-        }
-
-        It "Default OpenSSL version is 1.1" {
-            $commandResult = Get-CommandResult "openssl version"
-            $commandResult.Output | Should -Match "OpenSSL 1.1"
-        }
-    }
-
     It "PostgreSQL-Client" {
         "psql --version" | Should -ReturnZeroExitCode
     }
@@ -256,10 +201,6 @@ Describe "Common utilities" {
         "pg_config --version" | Should -ReturnZeroExitCode
     }
 
-    It "Azcopy" {
-        "azcopy --version" | Should -ReturnZeroExitCode
-    }
-
     It "PHP" {
         Get-WhichTool "php" | Should -Not -BeLike "/usr/bin/php*"
         "php --version" | Should -ReturnZeroExitCode
@@ -314,10 +255,6 @@ Describe "Haskell" -Skip:($os.IsHighSierra) {
     It "Cabal" {
         "cabal --version" | Should -ReturnZeroExitCode
     }
-
-    It "Stack" {
-        "stack --version" | Should -ReturnZeroExitCode
-    }
 }
 
 Describe "GCC" -Skip:($os.IsHighSierra) {
@@ -370,3 +307,71 @@ Describe "Visual Studio For Mac" {
         $vstoolPath | Should -Exist
     }
 }
+
+Describe "AWS" {
+    It "AWS CLI" {
+        "aws --version" | Should -ReturnZeroExitCode
+    }
+    It "AWS SAM CLI" {
+        "sam --version" | Should -ReturnZeroExitCode
+    }
+
+    It "AWS Session Manager CLI" {
+        "session-manager-plugin --version" | Should -ReturnZeroExitCode
+    }
+}
+
+Describe "AzCopy" {
+    It "AzCopy" {
+        "azcopy --version" | Should -ReturnZeroExitCode
+    }
+}
+
+Describe "AzureCLI" {
+    It "Azure CLI" {
+        "az -v" | Should -ReturnZeroExitCode
+    }
+}
+
+Describe "Miniconda" {
+    It "Conda" {
+        Get-EnvironmentVariable "CONDA" | Should -Not -BeNullOrEmpty
+        $condaBinPath = Join-Path $env:CONDA "bin" "conda"
+        "$condaBinPath --version" | Should -ReturnZeroExitCode
+    }
+}
+
+Describe "AppCenterCLI" {
+    It "App Center CLI" {
+        "appcenter --version" | Should -ReturnZeroExitCode
+    }
+}
+
+Describe "OpenSSL" {
+    It "OpenSSL is available" {
+        "openssl version" | Should -ReturnZeroExitCode
+    }
+
+    It "OpenSSL 1.1 path exists" {
+        $openSSLpath = "/usr/local/opt/openssl@1.1"
+        $openSSLpath | Should -Exist
+    }
+
+    It "Default OpenSSL version is 1.1" {
+        $commandResult = Get-CommandResult "openssl version"
+        $commandResult.Output | Should -Match "OpenSSL 1.1"
+    }
+}
+
+Describe "Stack" -Skip:($os.IsHighSierra) {
+    It "Stack" {
+        "stack --version" | Should -ReturnZeroExitCode
+    }
+}
+
+Describe "Mongo" {
+    It "mongodb" {
+        "mongo --version" | Should -ReturnZeroExitCode
+        "mongod --version"| Should -ReturnZeroExitCode
+    }
+}

images/macos/tests/Java.Tests.ps1

@@ -57,4 +57,28 @@ Describe "Java" {
             }
         }
     }
+
+    Context "Maven" {
+        Describe "Maven" {
+            It "Maven" {
+                "mvn --version" | Should -ReturnZeroExitCode
+            }
+        }
+    }
+    
+    Context "Gradle" {
+        Describe "Gradle" {
+            It "Gradle is installed" {
+                "gradle --version" | Should -ReturnZeroExitCode
+            }
+        
+            It "Gradle is installed to /usr/local/bin" {
+                (Get-Command "gradle").Path | Should -BeExactly "/usr/local/bin/gradle"
+            }
+        
+            It "Gradle is compatible with init.d plugins" {
+                "cd /tmp && gradle tasks" | Should -ReturnZeroExitCode
+            }
+        }
+    }
 }

images/macos/tests/XamarinNDK.Tests.ps1

@@ -0,0 +1,32 @@
+
+Import-Module "$PSScriptRoot/../helpers/Common.Helpers.psm1"
+Import-Module "$PSScriptRoot/../helpers/Tests.Helpers.psm1"
+Import-Module "$PSScriptRoot/../software-report/SoftwareReport.Android.psm1"
+
+$os = Get-OSVersion
+
+Describe "Xamarin NDK" {
+    BeforeAll {
+        $androidNdkToolchains = @("mips64el-linux-android-4.9", "mipsel-linux-android-4.9")
+        $ANDROID_SDK_DIR = Join-Path $env:HOME "Library" "Android" "sdk"
+    }
+    
+    Context "Xamarin NDK toolchains" -Skip:($os.IsBigSur) {
+        $testCases = $androidNdkToolchains | ForEach-Object { @{AndroidNdkToolchain = $_} }
+
+        It "<AndroidNdkToolchain>" -TestCases $testCases {
+            param ([string] $AndroidNdkToolchain)
+
+            $toolchainPath = Join-Path $ANDROID_SDK_DIR "ndk-bundle" "toolchains" $AndroidNdkToolchain
+            $toolchainPath | Should -Exist
+        }
+    }
+
+    Context "Xamarin Legacy NDK versions" -Skip:($os.IsBigSur) {
+        It "Android NDK version r18b is installed" {
+            $ndk18BundlePath = Join-Path $ANDROID_SDK_DIR "ndk" "18.1.5063045" "source.properties"
+            $rawContent = Get-Content $ndk18BundlePath -Raw
+            $rawContent | Should -BeLikeExactly "*Revision = 18.*"
+        }
+    }
+}