[macOS] change Invoke-Pester function (#2261)

* [macOS] change Invoke-Pester function

* added new logic

* try another approach.

* invoke-tests change

* added a little bunch of tests.

* remove import from browsers

* added source for all tests covered scripts.

* run all tests.

* added proper module import

* RunAll-Tests.ps1 changes.

* added shebang

* added new approach for tests.

* the first line should be shebang

* fixed nitpicks

images.CI/macos/azure-pipelines/image-generation.yml

@@ -121,15 +121,6 @@ jobs:
       arguments: -PackerLogPath "$(Agent.TempDirectory)/packer-log.txt" `
                         -PrintTopNLongest 25
 
-  - task: PublishTestResults@2
-    inputs:
-      testResultsFiles: '*.xml'
-      testResultsFormat: NUnit
-      searchFolder: '$(Common.TestResultsDirectory)'
-      failTaskOnFailedTests: true
-    displayName: Publish test results
-    condition: always()
-
   - task: PowerShell@2
     displayName: 'Move vm to cold storage and clear datastore tag'
     condition: always()

images/macos/helpers/Tests.Helpers.psm1

@@ -1,8 +1,4 @@
-# Invokes command and validate that the exit code is 0
-function Validate-ZeroExitCode($command) {
-    $result = Get-CommandResult $command
-    $result.ExitCode | Should -Be 0 -Because $result.Output
-}
+Import-Module "$PSScriptRoot/Common.Helpers.psm1"
 
 # Validates that tool is installed and in PATH
 function Validate-ToolExist($tool) {
@@ -85,3 +81,39 @@ function ShouldReturnZeroExitCode {
 If (Get-Command -Name Add-AssertionOperator -ErrorAction SilentlyContinue) {
     Add-AssertionOperator -Name ReturnZeroExitCode -InternalName ShouldReturnZeroExitCode -Test ${function:ShouldReturnZeroExitCode}
 }
+
+function Invoke-PesterTests {
+    Param(
+        [Parameter(Mandatory)][string] $TestFile,
+        [string] $TestName
+    )
+
+    $testPath = "$HOME/image-generation/tests/${TestFile}.Tests.ps1"
+    if (-not (Test-Path $testPath)) {
+        throw "Unable to find test file '$TestFile' on '$testPath'."
+    }
+
+    # Check that Pester module is imported
+    if (!(Get-Module "Pester")) {
+        Import-Module Pester
+    }
+
+    $configuration = [PesterConfiguration] @{
+        Run = @{ Path = $testPath; PassThru = $true }
+        Output = @{ Verbosity = "Detailed" }
+    }
+    if ($TestName) {
+        $configuration.Filter.FullName = $TestName
+    }
+
+    # Switch ErrorActionPreference to Stop temporary to make sure that tests will on silent errors too
+    $backupErrorActionPreference = $ErrorActionPreference
+    $ErrorActionPreference = "Stop"
+    $results = Invoke-Pester -Configuration $configuration
+    $ErrorActionPreference = $backupErrorActionPreference
+
+    # Fail in case if no tests are run
+    if (-not ($results -and ($results.FailedCount -eq 0) -and ($results.PassedCount -gt 0))) {
+        $results
+    }
+}

images/macos/provision/core/android-toolsets.sh

@@ -1,5 +1,6 @@
 #!/bin/bash -e -o pipefail
 source ~/utils/utils.sh
+source ~/utils/invoke-tests.sh
 
 function filter_components_by_version {
     minimumVersion=$1
@@ -105,3 +106,5 @@ curl -L -o proguard5.tgz $PROGUARD_LOCATION
 tar xzf proguard5.tgz --strip 1 && rm -f proguard5.tgz
 cp ../proguard4/proguard-*.txt . # Copy the Proguard Android definitions from the previous version
 popd
+
+invoke_tests "Android"

images/macos/provision/core/audiodevice.sh

@@ -1,4 +1,5 @@
 #!/bin/bash -e -o pipefail
+source ~/utils/invoke-tests.sh
 
 echo "install soundflower"
 brew cask install soundflower
@@ -12,3 +13,5 @@ brew install sox
 echo "set Soundflower (2ch) as input/output device"
 SwitchAudioSource -s "Soundflower (2ch)" -t input
 SwitchAudioSource -s "Soundflower (2ch)" -t output
+
+invoke_tests "Common" "Audio Device"

images/macos/provision/core/aws.sh

@@ -1,6 +1,7 @@
 #!/bin/bash -e -o pipefail
 
 source ~/utils/utils.sh
+source ~/utils/invoke-tests.sh
 
 echo Installing aws...
 AWS_CLI_URL="https://awscli.amazonaws.com/AWSCLIV2.pkg"
@@ -13,3 +14,5 @@ brew install aws-sam-cli
 
 echo "Install aws cli session manager"
 brew cask install session-manager-plugin
+
+invoke_tests "Common" "AWS"

images/macos/provision/core/azcopy.sh

@@ -1,6 +1,7 @@
 #!/bin/bash -e -o pipefail
 
 source ~/utils/utils.sh
+source ~/utils/invoke-tests.sh
 
 AZCOPY_DOWNLOAD_URL="https://aka.ms/downloadazcopy-v10-mac"
 
@@ -12,3 +13,5 @@ chmod +x "/usr/local/bin/azcopy"
 
 echo "Done, cleaning up"
 rm -rf azcopy*
+
+invoke_tests "Common" "Azcopy"

images/macos/provision/core/chrome.sh

@@ -1,5 +1,6 @@
 #!/bin/bash -e -o pipefail
 source ~/utils/utils.sh
+source ~/utils/invoke-tests.sh
 
 echo "Installing Chrome..."
 brew_cask_install_ignoring_sha256 "google-chrome"
@@ -12,3 +13,5 @@ brew install selenium-server-standalone
 
 CHROMEWEBDRIVER_DIR=$(readlink $(which chromedriver) | xargs dirname)
 echo "export CHROMEWEBDRIVER=$CHROMEWEBDRIVER_DIR" >> "${HOME}/.bashrc"
+
+invoke_tests "Browsers" "Chrome"

images/macos/provision/core/edge.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -e -o pipefail
-
 source ~/utils/utils.sh
+source ~/utils/invoke-tests.sh
 
 echo "Installing Microsoft Edge..."
 brew cask install microsoft-edge
@@ -58,3 +58,5 @@ sudo rm -rf "$HOME/Library/Caches/com.microsoft.autoupdate2"
 sudo rm -rf "/Library/Application Support/Microsoft/MAU2.0/"
 sudo rm -rf "/Library/LaunchAgents/com.microsoft.update.agent.plist"
 sudo rm -rf "/Library/PrivelegedHelperTools/com.microsoft.autoupdate.helper"
+
+invoke_tests "Browsers" "Edge"

images/macos/provision/core/firefox.sh

@@ -1,4 +1,5 @@
 #!/bin/bash -e -o pipefail
+source ~/utils/invoke-tests.sh
 
 echo "Installing Firefox..."
 brew cask install firefox
@@ -8,3 +9,5 @@ brew install geckodriver
 
 echo "Add GECKOWEBDRIVER to bashrc..."
 echo "export GECKOWEBDRIVER=$(brew --prefix geckodriver)/bin" >> "${HOME}/.bashrc"
+
+invoke_tests "Browsers" "Firefox"

images/macos/provision/core/powershell.sh

@@ -1,6 +1,7 @@
 #!/bin/bash -e -o pipefail
 
 source ~/utils/utils.sh
+source ~/utils/invoke-tests.sh
 
 echo Installing Azure CLI...
 brew install azure-cli
@@ -34,3 +35,5 @@ sudo ln -s /usr/local/bin/pwsh /usr/local/bin/powershell
 
 # fix ~/.azure directory permissions
 sudo chown -R ${USER}: $HOME/.azure
+
+invoke_tests "Powershell"

images/macos/provision/utils/invoke-tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -e -o pipefail
+
+invoke_tests() {
+    local TEST_FILE="$1"
+    local TEST_NAME="$2"
+
+    pwsh -Command "Import-Module '$HOME/image-generation/helpers/Tests.Helpers.psm1' -DisableNameChecking
+        Invoke-PesterTests -TestFile \"$TEST_FILE\" -TestName \"$TEST_NAME\""
+}

images/macos/templates/macOS-10.13.json

@@ -126,15 +126,15 @@
             "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}",
             "pause_before": "30s",
             "scripts": [
-                "./provision/core/xcode-clt.sh",
                 "./provision/core/homebrew.sh",
+                "./provision/core/powershell.sh",
+                "./provision/core/xcode-clt.sh",
                 "./provision/core/dotnet.sh",
                 "./provision/core/python.sh",
                 "./provision/core/azcopy.sh",
                 "./provision/core/openssl.sh",
                 "./provision/core/ruby.sh",
                 "./provision/core/rubygem.sh",
-                "./provision/core/powershell.sh",
                 "./provision/core/git.sh",
                 "./provision/core/node.sh"
             ]
@@ -185,7 +185,7 @@
             "type": "shell",
             "inline": [
                 "pwsh -File \"$HOME/image-generation/software-report/SoftwareReport.Generator.ps1\" -OutputDirectory \"$HOME/image-generation/output/software-report\" -ImageName {{user `build_id`}}",
-                "pwsh -File \"$HOME/image-generation/tests/Run-Tests.ps1\" -WorkingDirectory \"$HOME/image-generation/tests\" -OutputFile \"$HOME/image-generation/output/tests/test-results.xml\""
+                "pwsh -File \"$HOME/image-generation/tests/RunAll-Tests.ps1\""
             ]
         },
         {

images/macos/templates/macOS-10.14.json

@@ -126,15 +126,15 @@
             "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}",
             "pause_before": "30s",
             "scripts": [
-                "./provision/core/xcode-clt.sh",
                 "./provision/core/homebrew.sh",
+                "./provision/core/powershell.sh",
+                "./provision/core/xcode-clt.sh",
                 "./provision/core/dotnet.sh",
                 "./provision/core/python.sh",
                 "./provision/core/azcopy.sh",
                 "./provision/core/openssl.sh",
                 "./provision/core/ruby.sh",
                 "./provision/core/rubygem.sh",
-                "./provision/core/powershell.sh",
                 "./provision/core/git.sh",
                 "./provision/core/node.sh"
             ]
@@ -201,7 +201,7 @@
             "type": "shell",
             "inline": [
                 "pwsh -File \"$HOME/image-generation/software-report/SoftwareReport.Generator.ps1\" -OutputDirectory \"$HOME/image-generation/output/software-report\" -ImageName {{user `build_id`}}",
-                "pwsh -File \"$HOME/image-generation/tests/Run-Tests.ps1\" -WorkingDirectory \"$HOME/image-generation/tests\" -OutputFile \"$HOME/image-generation/output/tests/test-results.xml\""
+                "pwsh -File \"$HOME/image-generation/tests/RunAll-Tests.ps1\""
             ]
         },
         {

images/macos/templates/macOS-10.15.json

@@ -127,15 +127,15 @@
             "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}",
             "pause_before": "30s",
             "scripts": [
-                "./provision/core/xcode-clt.sh",
                 "./provision/core/homebrew.sh",
+                "./provision/core/powershell.sh",
+                "./provision/core/xcode-clt.sh",
                 "./provision/core/dotnet.sh",
                 "./provision/core/python.sh",
                 "./provision/core/azcopy.sh",
                 "./provision/core/openssl.sh",
                 "./provision/core/ruby.sh",
                 "./provision/core/rubygem.sh",
-                "./provision/core/powershell.sh",
                 "./provision/core/git.sh",
                 "./provision/core/node.sh"
             ]
@@ -202,7 +202,7 @@
             "type": "shell",
             "inline": [
                 "pwsh -File \"$HOME/image-generation/software-report/SoftwareReport.Generator.ps1\" -OutputDirectory \"$HOME/image-generation/output/software-report\" -ImageName {{user `build_id`}}",
-                "pwsh -File \"$HOME/image-generation/tests/Run-Tests.ps1\" -WorkingDirectory \"$HOME/image-generation/tests\" -OutputFile \"$HOME/image-generation/output/tests/test-results.xml\""
+                "pwsh -File \"$HOME/image-generation/tests/RunAll-Tests.ps1\""
             ]
         },
         {

images/macos/templates/macOS-11.0.json

@@ -127,15 +127,15 @@
             "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}",
             "pause_before": "30s",
             "scripts": [
-                "./provision/core/xcode-clt.sh",
                 "./provision/core/homebrew.sh",
+                "./provision/core/powershell.sh",
+                "./provision/core/xcode-clt.sh",
                 "./provision/core/dotnet.sh",
                 "./provision/core/python.sh",
                 "./provision/core/azcopy.sh",
                 "./provision/core/openssl.sh",
                 "./provision/core/ruby.sh",
                 "./provision/core/rubygem.sh",
-                "./provision/core/powershell.sh",
                 "./provision/core/git.sh",
                 "./provision/core/node.sh"
             ]
@@ -198,7 +198,7 @@
             "type": "shell",
             "inline": [
                 "pwsh -File \"$HOME/image-generation/software-report/SoftwareReport.Generator.ps1\" -OutputDirectory \"$HOME/image-generation/output/software-report\" -ImageName {{user `build_id`}}",
-                "pwsh -File \"$HOME/image-generation/tests/Run-Tests.ps1\" -WorkingDirectory \"$HOME/image-generation/tests\" -OutputFile \"$HOME/image-generation/output/tests/test-results.xml\""
+                "pwsh -File \"$HOME/image-generation/tests/RunAll-Tests.ps1\""
             ]
         },
         {

images/macos/tests/Browsers.Tests.ps1

@@ -0,0 +1,34 @@
+Describe "Chrome" {
+    It "Chrome" {
+        $chromeLocation = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+        $chromeLocation | Should -Exist
+        "'$chromeLocation' --version" | Should -ReturnZeroExitCode
+    }
+
+    It "Chrome Driver" {
+        "chromedriver --version" | Should -ReturnZeroExitCode
+    }
+}
+
+Describe "Edge" {
+    It "Microsoft Edge" {
+        $edgeLocation = "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"
+        $edgeLocation | Should -Exist
+        "'$edgeLocation' --version" | Should -ReturnZeroExitCode
+    }
+
+    It "Microsoft Edge Driver" {
+        "msedgedriver --version" | Should -ReturnZeroExitCode
+    }
+}
+
+Describe "Firefox" {
+    It "Firefox" {
+        $firefoxLocation = "/Applications/Firefox.app/Contents/MacOS/firefox"
+        $firefoxLocation | Should -Exist
+        "'$firefoxLocation' --version" | Should -ReturnZeroExitCode
+    }
+    It "Geckodriver" {
+        "geckodriver --version" | Should -ReturnZeroExitCode
+    }
+}

images/macos/tests/Common.Tests.ps1

@@ -254,38 +254,6 @@ Describe "Common utilities" {
     }
 }
 
-Describe "Browsers" {
-    It "Chrome" {
-        $chromeLocation = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
-        $chromeLocation | Should -Exist
-        "'$chromeLocation' --version" | Should -ReturnZeroExitCode
-    }
-
-    It "Chrome Driver" {
-        "chromedriver --version" | Should -ReturnZeroExitCode
-    }
-
-    It "Microsoft Edge" {
-        $edgeLocation = "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"
-        $edgeLocation | Should -Exist
-        "'$edgeLocation' --version" | Should -ReturnZeroExitCode
-    }
-
-    It "Microsoft Edge Driver" {
-        "msedgedriver --version" | Should -ReturnZeroExitCode
-    }
-
-    It "Firefox" {
-        $firefoxLocation = "/Applications/Firefox.app/Contents/MacOS/firefox"
-        $firefoxLocation | Should -Exist
-        "'$firefoxLocation' --version" | Should -ReturnZeroExitCode
-    }
-
-    It "Geckodriver" {
-        "geckodriver --version" | Should -ReturnZeroExitCode
-    }
-}
-
 Describe "Rust" -Skip:($os.IsHighSierra) {
     It "Rustup is installed" {
         "rustup --version" | Should -ReturnZeroExitCode

images/macos/tests/Run-Tests.ps1

@@ -1,25 +0,0 @@
-param (
-    [Parameter(Mandatory=$true)]
-    $WorkingDirectory,
-    [Parameter(Mandatory=$true)]
-    $OutputFile
-)
-
-$ErrorActionPreference = "Stop"
-
-Import-Module Pester
-Set-Location -Path $WorkingDirectory
-
-$parentDirectory = Split-Path $OutputFile -Parent
-if (-not (Test-Path $parentDirectory)) { New-Item -Path $parentDirectory -ItemType Directory | Out-Null }
-
-$configuration = [PesterConfiguration] @{
-    Output = @{ Verbosity = "Detailed" }
-    TestResult = @{
-        Enabled = $true
-        OutputPath = $OutputFile;
-        OutputFormat = "NUnitXML"
-    }
-}
-
-Invoke-Pester -Configuration $configuration

images/macos/tests/RunAll-Tests.ps1

@@ -0,0 +1,3 @@
+Import-Module "$PSScriptRoot/../helpers/Tests.Helpers.psm1"
+
+Invoke-PesterTests "*"