Merge main into install-codeql-bundle.

help/CreateImageAndAzureResources.md

@@ -12,6 +12,7 @@ After successful image generation, a snapshot of the temporary VM will be conver
 - `packer` - Can be downloaded from https://www.packer.io/downloads
 - `PowerShell 5.0 or higher` or `PSCore` for linux distributes.
 - `Azure CLI ` - https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest
+- `Azure Powershell module` - https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-4.6.1
 
 ### Azure DevOps self-hosted pool requirements
 To connect to a temporary VM packer use WinRM or SSH connections on public IP interfaces.
@@ -31,7 +32,7 @@ Download `packer` from https://www.packer.io/downloads, or install it via Chocol
 choco install packer
 ```
 
-Install Azure CLI - https://docs.microsoft.com/ru-ru/cli/azure/install-azure-cli-windows?view=azure-cli-latest&tabs=azure-cli.
+Install Azure CLI - https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-windows?view=azure-cli-latest&tabs=azure-cli.
 ```
 Invoke-WebRequest -Uri https://aka.ms/installazurecliwindows -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; rm .\AzureCLI.msi
 ```

helpers/GenerateResourcesAndImage.ps1

@@ -110,12 +110,12 @@ Function GenerateResourcesAndImage {
     $ServicePrincipalClientSecret = $env:UserName + [System.GUID]::NewGuid().ToString().ToUpper();
     $InstallPassword = $env:UserName + [System.GUID]::NewGuid().ToString().ToUpper();
 
-    Login-AzureRmAccount
+    Connect-AzAccount
-    Set-AzureRmContext -SubscriptionId $SubscriptionId
+    Set-AzContext -SubscriptionId $SubscriptionId
 
     $alreadyExists = $true;
     try {
-        Get-AzureRmResourceGroup -Name $ResourceGroupName
+        Get-AzResourceGroup -Name $ResourceGroupName
         Write-Verbose "Resource group was found, will delete and recreate it."
     }
     catch {
@@ -126,8 +126,8 @@ Function GenerateResourcesAndImage {
     if ($alreadyExists) {
         if($Force -eq $true) {
             # Cleanup the resource group if it already exitsted before
-            Remove-AzureRmResourceGroup -Name $ResourceGroupName -Force
+            Remove-AzResourceGroup -Name $ResourceGroupName -Force
-            New-AzureRmResourceGroup -Name $ResourceGroupName -Location $AzureLocation
+            New-AzResourceGroup -Name $ResourceGroupName -Location $AzureLocation
         } else {
             $title = "Delete Resource Group"
             $message = "The resource group you specified already exists. Do you want to clean it up?"
@@ -146,13 +146,13 @@ Function GenerateResourcesAndImage {
 
             switch ($result)
             {
-                0 { Remove-AzureRmResourceGroup -Name $ResourceGroupName -Force; New-AzureRmResourceGroup -Name $ResourceGroupName -Location $AzureLocation }
+                0 { Remove-AzResourceGroup -Name $ResourceGroupName -Force; New-AzResourceGroup -Name $ResourceGroupName -Location $AzureLocation }
                 1 { <# Do nothing #> }
                 2 { exit }
             }
         }
     } else {
-        New-AzureRmResourceGroup -Name $ResourceGroupName -Location $AzureLocation
+        New-AzResourceGroup -Name $ResourceGroupName -Location $AzureLocation
     }
 
     # This script should follow the recommended naming conventions for azure resources
@@ -164,19 +164,19 @@ Function GenerateResourcesAndImage {
     $storageAccountName = $storageAccountName.Replace("-", "").Replace("_", "").Replace("(", "").Replace(")", "").ToLower()
     $storageAccountName += "001"
 
-    New-AzureRmStorageAccount -ResourceGroupName $ResourceGroupName -AccountName $storageAccountName -Location $AzureLocation -SkuName "Standard_LRS"
+    New-AzStorageAccount -ResourceGroupName $ResourceGroupName -AccountName $storageAccountName -Location $AzureLocation -SkuName "Standard_LRS"
 
     $spDisplayName = [System.GUID]::NewGuid().ToString().ToUpper()
-    $sp = New-AzureRmADServicePrincipal -DisplayName $spDisplayName -Password (ConvertTo-SecureString $ServicePrincipalClientSecret -AsPlainText -Force)
+    $sp = New-AzADServicePrincipal -DisplayName $spDisplayName -Password (ConvertTo-SecureString $ServicePrincipalClientSecret -AsPlainText -Force)
 
     $spAppId = $sp.ApplicationId
     $spClientId = $sp.ApplicationId
     $spObjectId = $sp.Id
     Start-Sleep -Seconds $SecondsToWaitForServicePrincipalSetup
 
-    New-AzureRmRoleAssignment -RoleDefinitionName Contributor -ServicePrincipalName $spAppId
+    New-AzRoleAssignment -RoleDefinitionName Contributor -ServicePrincipalName $spAppId
     Start-Sleep -Seconds $SecondsToWaitForServicePrincipalSetup
-    $sub = Get-AzureRmSubscription -SubscriptionId $SubscriptionId
+    $sub = Get-AzSubscription -SubscriptionId $SubscriptionId
     $tenantId = $sub.TenantId
     # "", "Note this variable-setting script for running Packer with these Azure resources in the future:", "==============================================================================================", "`$spClientId = `"$spClientId`"", "`$ServicePrincipalClientSecret = `"$ServicePrincipalClientSecret`"", "`$SubscriptionId = `"$SubscriptionId`"", "`$tenantId = `"$tenantId`"", "`$spObjectId = `"$spObjectId`"", "`$AzureLocation = `"$AzureLocation`"", "`$ResourceGroupName = `"$ResourceGroupName`"", "`$storageAccountName = `"$storageAccountName`"", "`$install_password = `"$install_password`"", ""
 

images.CI/macos/azure-pipelines/image-generation.yml

@@ -7,6 +7,8 @@ jobs:
   variables:
   - group: Mac-Cloud Image Generation
   - group: Mac-Cloud Image Generation Key Vault
+  - name: VirtualMachineName
+    value: $(Build.BuildNumber).$(System.JobAttempt)
 
   steps:
   - checkout: self
@@ -56,7 +58,7 @@ jobs:
         -var="output_folder=$(output-folder)" `
         -var="vm_username=$(vm-username)" `
         -var="vm_password=$(vm-password)" `
-        -var="build_id=$(Build.BuildNumber)" `
+        -var="build_id=${{ variables.VirtualMachineName }}" `
         -var="baseimage_name=${{ parameters.base_image_name }}" `
         -var="github_feed_token=$(github-feed-token)" `
         -var="xcode_install_user=$(xcode-installation-user)" `
@@ -83,7 +85,7 @@ jobs:
       ls $(Common.TestResultsDirectory)
 
       echo "Put VM name to 'VM_Done_Name' file"
-      echo "$(Build.BuildNumber)" > "$(Build.ArtifactStagingDirectory)/VM_Done_Name"
+      echo "${{ variables.VirtualMachineName }}" > "$(Build.ArtifactStagingDirectory)/VM_Done_Name"
     displayName: Prepare artifact
 
   - bash: |

images/linux/scripts/installers/post-deployment.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+################################################################################
+##  File: post-deployment.sh
+##  Desc: Post deployment actions
+################################################################################
+
+# set chmod -R 777 /opt
+if [[ -d "/opt" ]]; then
+    echo "chmod -R 777 /opt"
+    chmod -R 777 /opt
+fi
+
+# remove installer and helper folders
+rm -rf $HELPER_SCRIPT_FOLDER
+rm -rf $INSTALLER_SCRIPT_FOLDER
+chmod 755 $IMAGE_FOLDER

images/linux/ubuntu1604.json

@@ -304,10 +304,13 @@
         },
         {
             "type": "shell",
-            "inline": [
+            "scripts":[
-                "rm -rf {{user `helper_script_folder`}}",
+                "{{template_dir}}/scripts/installers/post-deployment.sh"
-                "rm -rf {{user `installer_script_folder`}}",
+            ],
-                "chmod 755 {{user `image_folder`}}"
+            "environment_vars":[
+                "HELPER_SCRIPT_FOLDER={{user `helper_script_folder`}}",
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
+                "IMAGE_FOLDER={{user `image_folder`}}"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },

images/linux/ubuntu1804.json

@@ -308,10 +308,13 @@
         },
         {
             "type": "shell",
-            "inline": [
+            "scripts":[
-                "rm -rf {{user `helper_script_folder`}}",
+                "{{template_dir}}/scripts/installers/post-deployment.sh"
-                "rm -rf {{user `installer_script_folder`}}",
+            ],
-                "chmod 755 {{user `image_folder`}}"
+            "environment_vars":[
+                "HELPER_SCRIPT_FOLDER={{user `helper_script_folder`}}",
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
+                "IMAGE_FOLDER={{user `image_folder`}}"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },

images/linux/ubuntu2004.json

@@ -308,6 +308,18 @@
             "destination": "{{template_dir}}/Ubuntu2004-README.md",
             "direction": "download"
         },
+        {
+            "type": "shell",
+            "scripts":[
+                "{{template_dir}}/scripts/installers/post-deployment.sh"
+            ],
+            "environment_vars":[
+                "HELPER_SCRIPT_FOLDER={{user `helper_script_folder`}}",
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
+                "IMAGE_FOLDER={{user `image_folder`}}"
+            ],
+            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+        },
         {
             "type": "shell",
             "scripts":[
@@ -317,15 +329,6 @@
                 "RUN_VALIDATION={{user `run_validation_diskspace`}}"
             ]
         },
-        {
-            "type": "shell",
-            "inline": [
-                "rm -rf {{user `helper_script_folder`}}",
-                "rm -rf {{user `installer_script_folder`}}",
-                "chmod 755 {{user `image_folder`}}"
-            ],
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
-        },
         {
             "type": "file",
             "source": "{{template_dir}}/config/ubuntu2004.conf",

images/macos/helpers/Xcode.Helpers.psm1

@@ -91,4 +91,17 @@ function Get-XcodePairsList {
         $result += "$watchName $phoneName"
     }
     return $result
+}
+
+function Test-XcodeStableVersion {
+    param([Parameter(Mandatory)][string]$Version)
+
+    if ($Version -match "beta") {
+        return $false
+    }
+    if ($Version -match "GM") {
+        return $false
+    }
+    
+    return $true
 }

images/macos/provision/configuration/configure-machine.sh

@@ -18,4 +18,13 @@ sudo pmset hibernatemode 0
 sudo rm -f /var/vm/sleepimage
 
 # Change screen resolution to the maximum supported for 4Mb video memory
-sudo "/Library/Application Support/VMware Tools/vmware-resolutionSet" 1176 885
+sudo "/Library/Application Support/VMware Tools/vmware-resolutionSet" 1176 885
+
+# https://developer.apple.com/support/expiration/
+# Enterprise iOS Distribution Certificates generated between February 7 and September 1st, 2020 will expire on February 7, 2023.
+# Rotate the certificate before expiration to ensure your apps are installed and signed with an active certificate.
+# Confirm that the correct intermediate certificate is installed by verifying the expiration date is set to 2030.
+# sudo security delete-certificate -Z FF6797793A3CD798DC5B2ABEF56F73EDC9F83A64 /Library/Keychains/System.keychain
+curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer --output $HOME/AppleWWDRCAG3.cer --silent
+sudo security add-trusted-cert -d -r unspecified -k /Library/Keychains/System.keychain $HOME/AppleWWDRCAG3.cer
+rm $HOME/AppleWWDRCAG3.cer

images/macos/provision/core/commonutils.sh

@@ -1,15 +1,14 @@
 #!/bin/sh
-
 set -e
 
 source ~/utils/utils.sh
+
+# TO-DO: Move the list of brew packages and casks to toolset
+
 # brew install
 binst_common_utils=(
     carthage
-    xctool
     cmake
-    bats
-    parallel
     subversion
     go
     gnupg
@@ -27,6 +26,14 @@ binst_common_utils=(
     aria2
 )
 
+if is_Less_BigSur; then
+    binst_common_utils+=(
+        xctool
+        bats
+        parallel
+    )
+fi
+
 for package in ${binst_common_utils[@]}; do
     echo "Install $package"
     brew install $package

images/macos/provision/core/dotnet.sh

@@ -19,7 +19,13 @@ chmod +x ./dotnet-install.sh
 ARGS_LIST=()
 echo "Parsing dotnet SDK (except rc and preview versions) from .json..."
 
-if is_Less_Catalina; then
+# TO-DO: move the list of versions to install to toolset
+if is_BigSur; then
+    DOTNET_CHANNELS=(
+    'https://raw.githubusercontent.com/dotnet/core/master/release-notes/2.1/releases.json'
+    'https://raw.githubusercontent.com/dotnet/core/master/release-notes/3.1/releases.json' 
+    )
+elif is_Less_Catalina; then
     DOTNET_CHANNELS=(
     'https://raw.githubusercontent.com/dotnet/core/master/release-notes/2.1/releases.json'
     )

images/macos/provision/core/node.sh

@@ -39,7 +39,9 @@ fi
 echo Installing yarn...
 curl -o- -L https://yarnpkg.com/install.sh | bash
 
-for module in ${node_common_modules[@]}; do
+if is_Less_BigSur; then
-    echo "Install $module"
+    for module in ${node_common_modules[@]}; do
-    npm install -g $module
+        echo "Install $module"
-done
+        npm install -g $module
+    done
+fi

images/macos/provision/core/openssl.sh

@@ -4,7 +4,11 @@ export PATH="/usr/local/bin:/usr/local/sbin:~/bin:$PATH"
 
 echo Installing OpenSSL...
 /usr/local/bin/brew install openssl
-/usr/local/bin/brew link openssl --force
+
+if is_BigSur; then
+    ln -sf $(brew --prefix openssl)/bin/openssl /usr/local/bin/openssl
+    exit 0
+fi
 
 # Install OpenSSL 1.0.2t
 # https://www.openssl.org/policies/releasestrat.html - Version 1.0.2 will be supported until 2019-12-31 (LTS)

images/macos/provision/core/python.sh

@@ -1,3 +1,6 @@
+#!/bin/sh
+source ~/utils/utils.sh
+
 echo "Installing Python Tooling"
 
 echo "Brew Installing Python 3"

images/macos/provision/core/ruby.sh

@@ -2,13 +2,16 @@
 
 set -e
 
-if [ $(id -u) -eq 0 ]; then
+source ~/utils/utils.sh
-    echo "This script can not run as root. Aborting..."
+
-    exit 1
+echo Installing Ruby...
+if is_Less_BigSur; then
+    # We can't install latest ruby 2.7 as a default version related with bug
+    # https://github.com/fastlane/fastlane/issues/15397
+    /usr/local/bin/brew install ruby@2.6
+    ln -sf /usr/local/opt/ruby\@2.6 /usr/local/opt/ruby
+else
+    brew install ruby
 fi
 
-# We can't install latest ruby 2.7 as a default version related with bug
+
-# https://github.com/fastlane/fastlane/issues/15397
-echo Installing Ruby...
-/usr/local/bin/brew install ruby@2.6
-ln -sf /usr/local/opt/ruby\@2.6 /usr/local/opt/ruby

images/macos/provision/core/rubygem.sh

@@ -10,14 +10,16 @@ gem install xcode-install --force
 echo Installing CocoaPods...
 gem install cocoapods
 
-# fix nomad-cli installation
+if is_Less_BigSur; then
-if is_HighSierra; then
+    # fix nomad-cli installation
-  brew install libxml2
+    if is_HighSierra; then
-  gem install nokogiri -v 1.6.8.1 -- --use-system-libraries --with-xml2-include=$(brew --prefix libxml2)/include/libxml2
+        brew install libxml2
-fi
+        gem install nokogiri -v 1.6.8.1 -- --use-system-libraries --with-xml2-include=$(brew --prefix libxml2)/include/libxml2
+    fi
 
-echo Installing nomad-cli...
+    echo Installing nomad-cli...
-gem install nomad-cli
+    gem install nomad-cli
+fi
 
 echo Installing xcpretty...
 gem install xcpretty

images/macos/provision/core/xcode-ctl.sh

@@ -9,7 +9,7 @@
 clt_placeholder="/tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress"
 touch $clt_placeholder
 clt_label=`/usr/sbin/softwareupdate -l |
-                      grep -B 1 -E 'Command Line Tools' |
+                      grep -B 1 -E 'Command Line Tools beta 5' |
                       awk -F'*' '/^ *\\*/ {print \$2}' |
                       sed -e 's/^ *Label: //' -e 's/^ *//' |
                       sort -V |

images/macos/provision/core/xcode-tools.sh

@@ -36,7 +36,12 @@ do
     extractXcodeXip $WORK_DIR "$VERSION_TO_INSTALL"
 
     # Remove "beta" postfix from version
-    XCODE_VERSION=$(echo $XCODE_VERSION | cut -d"_" -f 1)
+    if [[ $XCODE_VERSION == "12_beta" ]] && is_Catalina ; then
+        # trick to install Xcode 12 GM and Xcode 12 beta 6 side by side
+        XCODE_VERSION="12_beta"
+    else
+        XCODE_VERSION=$(echo $XCODE_VERSION | cut -d"_" -f 1)
+    fi
 
     echo "Checking if unpacked Xcode ${XCODE_VERSION} is valid"
     validateXcodeIntegrity "$WORK_DIR"
@@ -69,6 +74,13 @@ do
         continue
     fi
 
+    if [[ $XCODE_VERSION == "12_beta" ]] && is_Catalina ; then
+        # trick to install Xcode 12 GM and Xcode 12 beta 6 side by side
+        XCODE_VERSION="12_beta"
+    else
+        XCODE_VERSION=$(echo $XCODE_VERSION | cut -d"_" -f 1)
+    fi
+
     echo "Running 'runFirstLaunch' for Xcode ${XCODE_VERSION}..."
     runFirstLaunch $XCODE_VERSION
 done

images/macos/provision/utils/xcode-utils.sh

@@ -11,7 +11,7 @@ getXcodeVersionToInstall() {
     local XCODE_VERSION="$1"
 
     if  [[ ! $XCODE_VERSION =~ "_beta" ]]; then
-        echo "$XCODE_VERSION"
+        echo "${XCODE_VERSION//_/ }"
     else
         local XCODE_BETA="${XCODE_VERSION/_/ }"
         echo "$(xcversion list | sort -r | grep -m 1 "$XCODE_BETA")"
@@ -59,7 +59,6 @@ extractXcodeXip() {
     local WORKING_DIR="$1"
     local XCODE_VERSION="$2"
     XCODE_XIP="${WORKING_DIR}/Xcode_${XCODE_VERSION// /_}.xip"
-    echo "XCODE_XIP = $XCODE_XIP"
 
     open -W $XCODE_XIP
 
@@ -75,7 +74,7 @@ extractXcodeXip() {
 
 createBetaSymlink() {
     local XCODE_VERSION=$1
-    if [[ $XCODE_VERSION =~ 1[01].* ]] || [[ $XCODE_VERSION == "12" ]] || [[ $XCODE_VERSION == "12_beta" ]]; then
+    if [[ $XCODE_VERSION =~ 1[01].* ]] || [[ $XCODE_VERSION == "12" ]]; then
         ln -sf "/Applications/Xcode_${XCODE_VERSION}.app" "/Applications/Xcode_${XCODE_VERSION}_beta.app"
     fi
 }

images/macos/software-report/SoftwareReport.Common.psm1

@@ -58,7 +58,7 @@ function Get-GccVersion {
     $versionList = @("8", "9")
     $versionList | Foreach-Object {
         $version = Run-Command "gcc-${_} --version" | Select-Object -First 1
-        "$version  — available by ``gcc-${_}`` alias"
+        "$version - available by ``gcc-${_}`` alias"
     }
 }
 
@@ -66,7 +66,7 @@ function Get-FortranVersion {
     $versionList = @("8", "9")
     $versionList | Foreach-Object {
         $version = Run-Command "gfortran-${_} --version" | Select-Object -First 1
-        "$version  — available by ``gfortran-${_}`` alias"
+        "$version  - available by ``gfortran-${_}`` alias"
     }
 }
 

images/macos/software-report/SoftwareReport.Generator.ps1

@@ -77,7 +77,6 @@ $homebrewVersion = Run-Command "brew --version" | Select-Object -First 1
 $npmVersion = Run-Command "npm --version"
 $yarnVersion = Run-Command "yarn --version"
 $nugetVersion = Run-Command "nuget help" | Select-Object -First 1 | Take-Part -Part 2
-$pipVersion = Get-PipVersion -Version 2
 $pip3Version = Get-PipVersion -Version 3
 $condaVersion = Invoke-Expression "conda --version"
 $rubyGemsVersion = Run-Command "gem --version"
@@ -88,7 +87,13 @@ if ($os.IsHigherThanMojave) {
     $vcpkgVersion = Get-VcpkgVersion
     $markdown += New-MDList -Lines $vcpkgVersion -Style Unordered -NoNewLine
 }
+if ($os.IsLessThanBigSur) {
+    $pipVersion = Get-PipVersion -Version 2
+    $markdown += New-MDList -Style Unordered -Lines @("Pip ${pipVersion}") -NoNewLine
+}
+
 $markdown += New-MDList -Style Unordered -Lines @(
+    "Pip ${pip3Version}",
     $bundlerVersion,
     "Carthage ${carthageVersion}",
     "CocoaPods ${cocoaPodsVersion}",
@@ -96,8 +101,6 @@ $markdown += New-MDList -Style Unordered -Lines @(
     "NPM ${npmVersion}",
     "Yarn ${yarnVersion}",
     "NuGet ${nugetVersion}",
-    "Pip ${pipVersion}",
-    "Pip ${pip3Version}",
     "Mini${condaVersion}",
     "RubyGems ${rubyGemsVersion}",
     "Composer ${composerVersion}"
@@ -124,7 +127,6 @@ $gitLFSVersion = Run-Command "git-lfs version" | Take-Part -Part 0 | Take-Part -
 $hubVersion = Run-Command "hub version | grep 'hub version'" | Take-Part -Part 2
 $wgetVersion = Run-Command "wget --version" | Select-String "GNU Wget" | Take-Part -Part 2
 $svnVersion = Run-Command "svn --version --quiet"
-$parallelVersion = Run-Command "parallel --version" | Select-String "GNU parallel" | Select-Object -First 1
 $jqVersion = Run-Command "jq --version" | Take-Part -Part 1 -Delimiter "-"
 $opensslVersion = Get-Item /usr/local/opt/openssl | ForEach-Object {"{0} ``({1} -> {2})``" -f (Run-Command "openssl version"), $_.FullName, $_.Target}
 $gpgVersion = Run-Command "gpg --version" | Select-String 'gpg (GnuPG)' -SimpleMatch
@@ -153,7 +155,6 @@ $markdown += New-MDList -Style Unordered -NoNewLine -Lines @(
     "GNU Wget ${wgetVersion}",
     "Subversion (SVN) ${svnVersion}",
     "Packer $packerVersion",
-    $parallelVersion,
     $opensslVersion,
     "jq ${jqVersion}",
     $gpgVersion,
@@ -173,7 +174,11 @@ $markdown += New-MDList -Style Unordered -NoNewLine -Lines @(
 )
 if ($os.IsHigherThanMojave) {
     $newmanVersion = Run-Command "newman --version"
-    $markdown += New-MDList -Lines "Newman $newmanVersion" -Style Unordered
+    $markdown += New-MDList -Lines "Newman $newmanVersion" -Style Unordered -NoNewLine
+}
+if ($os.IsLessThanBigSur) {
+    $parallelVersion = Run-Command "parallel --version" | Select-String "GNU parallel" | Select-Object -First 1
+    $markdown += New-MDList -Lines $parallelVersion -Style Unordered
 }
 $markdown += New-MDNewLine
 

images/macos/software-report/SoftwareReport.Xcode.psm1

@@ -1,3 +1,7 @@
+Import-Module "$PSScriptRoot/../helpers/Common.Helpers.psm1"
+
+$os = Get-OSVersion
+
 function Get-XcodePaths {
     $xcodePaths = Get-ChildItemWithoutSymlinks "/Applications" -Filter "Xcode_*.app"
     return $xcodePaths | Select-Object -ExpandProperty Fullname
@@ -43,7 +47,7 @@ function Get-XcodeInfoList {
         $versionInfo.Path = $xcodeRootPath
         $versionInfo.IsDefault = ($xcodeRootPath -eq $defaultXcodeRootPath)
     
-        $xcodeInfo.Add($versionInfo.Version.ToString(), [PSCustomObject] @{
+        $xcodeInfo.Add($xcodeRootPath, [PSCustomObject] @{
             VersionInfo = $versionInfo
             SDKInfo = Get-XcodeSDKList
             SimulatorsInfo = Get-XcodeSimulatorsInfo
@@ -217,20 +221,27 @@ function Build-XcodeSimulatorsTable {
 }
 
 function Build-XcodeSupportToolsSection {
-    $nomadCLI = Run-Command "gem -v nomad-cli"
-    $nomadIPA = Run-Command "ipa -version"
     $xcpretty = Run-Command "xcpretty --version"
-    $xctool = Run-Command "xctool --version"
     $xcversion = Run-Command "xcversion --version" | Select-String "^[0-9]"
 
+    $toolList = @(
+        "xcpretty $xcpretty",
+        "xcversion $xcversion"
+    )
+
+    if ($os.IsLessThanBigSur) {
+        $nomadCLI = Run-Command "gem -v nomad-cli"
+        $nomadIPA = Run-Command "ipa -version"
+        $xctool = Run-Command "xctool --version"
+        $toolList += @(
+            "Nomad CLI $nomadCLI",
+            "Nomad CLI IPA $nomadIPA",
+            "xctool $xctool"
+        )
+    }
+
     $output = ""
     $output += New-MDHeader "Xcode Support Tools" -Level 4
-    $output += New-MDList -Style Unordered -Lines @(
+    $output += New-MDList -Style Unordered -Lines $toolList
-        "Nomad CLI $nomadCLI",
-        "Nomad CLI IPA $nomadIPA",
-        "xcpretty $xcpretty",
-        "xctool $xctool",
-        "xcversion $xcversion"
-    ) 
     return $output
 }

images/macos/templates/macOS-11.0.json

@@ -129,6 +129,7 @@
             "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}",
             "pause_before": "30s",
             "scripts": [
+                "./provision/core/xcode-ctl.sh",
                 "./provision/core/homebrew.sh",
                 "./provision/core/dotnet.sh",
                 "./provision/core/python.sh",

images/macos/tests/Common.Tests.ps1

@@ -12,6 +12,14 @@ Describe "Disk free space" {
     }
 }
 
+Describe "Certificate" {
+    It "Apple Worldwide Developer Relations Certification Authority[expired: 2030-02] is installed" {
+        $sha1Hash = "06EC06599F4ED0027CC58956B4D3AC1255114F35"
+        $certs = security find-certificate -a -c Worldwide -p -Z | Out-String
+        $certs | Should -Match $sha1Hash
+    }
+}
+
 Describe "Git" {
     It "git is installed" {
         "git --version" | Should -ReturnZeroExitCode
@@ -94,17 +102,19 @@ Describe "Common utilities" {
         "aliyun --version" | Should -ReturnZeroExitCode
     }
 
-    It "Mobile Center CLI" {
+    It "Mobile Center CLI" -Skip:($os.IsBigSur) {
         "mobile-center --version" | Should -ReturnZeroExitCode
     }
 
-    It "Nomad CLI" {
+    Context "Nomad" -Skip:($os.IsBigSur) {
-        $result = Get-CommandResult "gem list"
+        It "Nomad CLI" {
-        $result.Output | Should -BeLike "*nomad-cli*"
+            $result = Get-CommandResult "gem list"
-    }
+            $result.Output | Should -BeLike "*nomad-cli*"
-
+        }
-    It "Nomad CLI IPA" {
+    
-        "ipa --version" | Should -ReturnZeroExitCode
+        It "Nomad CLI IPA" {
+            "ipa --version" | Should -ReturnZeroExitCode
+        }
     }
 
     It "Conda" {
@@ -129,7 +139,7 @@ Describe "Common utilities" {
         "curl --version" | Should -ReturnZeroExitCode
     }
 
-    It "xctool" {
+    It "xctool" -Skip:($os.IsBigSur) {
         "xctool --version" | Should -ReturnZeroExitCode
     }
 
@@ -161,7 +171,7 @@ Describe "Common utilities" {
     It "PostgreSQL-Client" {
         "psql --version" | Should -ReturnZeroExitCode
     }
-    
+
     It "PostgreSQL-Server" {
         "pg_config --version" | Should -ReturnZeroExitCode
     }
@@ -178,11 +188,11 @@ Describe "Common utilities" {
         Get-WhichTool "php" | Should -Not -BeLike "/usr/bin/php*"
         "php --version" | Should -ReturnZeroExitCode
     }
-    
+
     It "Composer" {
         "composer --version" | Should -ReturnZeroExitCode
     }
-    
+
     It "R" -Skip:($os.IsBigSur) {
         "R --version" | Should -ReturnZeroExitCode
     }
@@ -198,7 +208,7 @@ Describe "Common utilities" {
     It "bazelisk" {
         "bazelisk version" | Should -ReturnZeroExitCode
     }
-    
+
     It "Julia" {
         "julia --version" | Should -ReturnZeroExitCode
     }
@@ -252,7 +262,7 @@ Describe "Browsers" {
     It "Microsoft Edge Driver" {
         "msedgedriver --version" | Should -ReturnZeroExitCode
     }
-    
+
     It "Firefox" {
         $firefoxLocation = "/Applications/Firefox.app/Contents/MacOS/firefox"
         $firefoxLocation | Should -Exist
@@ -304,7 +314,7 @@ Describe "Haskell" -Skip:($os.IsHighSierra) {
     It "GHC" {
         "ghc --version" | Should -ReturnZeroExitCode
     }
-    
+
     It "Cabal" {
         "cabal --version" | Should -ReturnZeroExitCode
     }
@@ -327,7 +337,7 @@ Describe "Gcc" -Skip:($os.IsHighSierra) {
         param (
             [string] $GccVersion
         )
-        
+
         "gcc-$GccVersion --version" | Should -ReturnZeroExitCode
     }
 }

images/macos/tests/Python.Tests.ps1

@@ -1,6 +1,8 @@
 Import-Module "$PSScriptRoot/../helpers/Common.Helpers.psm1"
 Import-Module "$PSScriptRoot/../helpers/Tests.Helpers.psm1"
 
+$os = Get-OSVersion
+
 Describe "Python" {
     It "Python 2 is available" {
         "python --version" | Should -ReturnZeroExitCode
@@ -10,7 +12,7 @@ Describe "Python" {
         (Get-CommandResult "python --version").Output | Should -BeLike "Python 2.*"
     }
 
-    It "Python 2 is installed under /usr/local/bin" {
+    It "Python 2 is installed under /usr/local/bin" -Skip:($os.IsBigSur) {
         Get-WhichTool "python" | Should -BeLike "/usr/local/bin*"
     }
 
@@ -22,7 +24,7 @@ Describe "Python" {
         Get-WhichTool "python3" | Should -BeLike "/usr/local/bin*"
     }
 
-    It "Pip 2 is available" {
+    It "Pip 2 is available" -Skip:($os.IsBigSur) {
         "pip --version" | Should -ReturnZeroExitCode
     }
 

images/macos/tests/Xamarin.Tests.ps1

@@ -191,7 +191,7 @@ Describe "Xamarin Bundles" {
         $XAMARIN_ANDROID_VERSIONS_PATH = "/Library/Frameworks/Xamarin.Android.framework/Versions"
     }
     
-    $XAMARIN_BUNDLES = Get-ToolsetValue "xamarin.bundles"
+    [array]$XAMARIN_BUNDLES = Get-ToolsetValue "xamarin.bundles"
     $XAMARIN_DEFAULT_BUNDLE = Get-ToolsetValue "xamarin.bundle-default"
     If ($XAMARIN_DEFAULT_BUNDLE -eq "latest") { $XAMARIN_DEFAULT_BUNDLE = $XAMARIN_BUNDLES[0].symlink }
     

images/macos/tests/Xcode.Tests.ps1

@@ -56,7 +56,7 @@ Describe "Xcode" {
     }
 
     Context "XCODE_DEVELOPER_DIR" {
-        $stableXcodeVersions = $XCODE_VERSIONS | Where-Object { -not ($_ -match "beta") }
+        $stableXcodeVersions = $XCODE_VERSIONS | Where-Object { Test-XcodeStableVersion $_ }
         $majorXcodeVersions = $stableXcodeVersions | ForEach-Object { $_.Split(".")[0] } | Select-Object -Unique
         $testCases = $majorXcodeVersions | ForEach-Object {
             $majorXcodeVersion = $_
@@ -84,7 +84,7 @@ Describe "Xcode" {
 }
 
 Describe "Xcode simulators" {
-    $XCODE_VERSIONS | ForEach-Object {
+    $XCODE_VERSIONS | Where-Object { Test-XcodeStableVersion $_ } | ForEach-Object {
         Switch-Xcode -Version $_
         
         Context "$_" {

images/macos/toolsets/toolset-10.15.json

@@ -2,11 +2,11 @@
     "xcode": {
         "default": "11.7",
         "versions": [
-            "12_beta", "11.7", "11.6", "11.5", "11.4.1", "11.4", "11.3.1", "11.2.1", "11.1", "11", "10.3"
+            "12.2_beta", "12", "12_beta", "11.7", "11.6", "11.5", "11.4.1", "11.4", "11.3.1", "11.2.1", "11.1", "11", "10.3"
         ]
     },
     "xamarin": {
-        "vsmac": "8.7.5.19",
+        "vsmac": "8.7.7.10",
         "mono-versions": [
             "6.12.0.93", "6.10.0.106", "6.8.0.123", "6.6.0.166", "6.4.0.208"
         ],

images/macos/toolsets/toolset-11.0.json

@@ -2,22 +2,22 @@
     "xcode": {
         "default": "11.7",
         "versions": [
-            "12_beta", "11.7"
+            "12.2_beta", "12_beta", "11.7"
         ]
     },
     "xamarin": {
-        "vsmac": "8.7.5.19",
+        "vsmac": "8.7.7.10",
         "mono-versions": [
-            "6.12.0.93", "6.10.0.106"
+            "6.12.0.93"
         ],
         "ios-versions": [
-            "13.20.2.2", "13.18.2.1"
+            "13.20.2.2"
         ],
         "mac-versions": [
-            "6.20.2.2", "6.18.3.2"
+            "6.20.2.2"
         ],
         "android-versions": [
-            "11.0.2.0", "10.3.1.4"
+            "11.0.2.0"
         ],
         "bundle-default": "latest",
         "bundles": [
@@ -27,13 +27,6 @@
                 "ios": "13.20",
                 "mac": "6.20",
                 "android": "11.0" 
-            },
-            {
-                "symlink": "6_10_0",
-                "mono":"6.10",
-                "ios": "13.18",
-                "mac": "6.18",
-                "android": "10.3" 
             }
         ]
     },
@@ -67,9 +60,6 @@
             "arch": "x64",
             "platform" : "darwin",
             "versions": [
-                "2.7.*",
-                "3.5.*",
-                "3.6.*",
                 "3.7.*",
                 "3.8.*"
             ]
@@ -91,7 +81,6 @@
             "arch": "x64",
             "platform" : "darwin",
             "versions": [
-                "1.14.*",
                 "1.15.*"
             ]
         }

images/win/scripts/Installers/Install-Edge.ps1

@@ -24,7 +24,14 @@ $EdgeDriverVersionFile = Start-DownloadWithRetry -Url $EdgeDriverVersionUrl -Nam
 Write-Host "Download Microsoft Edge WebDriver..."
 $EdgeDriverLatestVersion = Get-Content -Path $EdgeDriverVersionFile
 $EdgeDriverArchName = "edgedriver_win64.zip"
-$EdgeDriverDownloadUrl="https://msedgedriver.azureedge.net/${EdgeDriverLatestVersion}/${EdgeDriverArchName}"
+# A temporary workaround to install the previous driver version because 85.0.564.60 for win64 doesn't exist
+if ($EdgeDriverLatestVersion -eq "85.0.564.60")
+{
+    $EdgeDriverLatestVersion = "85.0.564.51"
+    Set-Content -Path $EdgeDriverVersionFile -Value $EdgeDriverLatestVersion
+}
+
+$EdgeDriverDownloadUrl = "https://msedgedriver.azureedge.net/${EdgeDriverLatestVersion}/${EdgeDriverArchName}"
 
 $EdgeDriverArchPath = Start-DownloadWithRetry -Url $EdgeDriverDownloadUrl -Name $EdgeDriverArchName
 

images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1

@@ -1,4 +1,11 @@
-function Get-AndroidComponentLocation {
+function Split-TableRowByColumns {
+    param(
+        [string] $Row
+    )
+    return $Row.Split("|") | ForEach-Object { $_.trim() }
+}
+
+function Get-AndroidSDKRoot {
     param(
         [string] $ComponentName
     )
@@ -6,24 +13,6 @@ function Get-AndroidComponentLocation {
     return "Location $path"
 }
 
-function Split-AndroidSDKOutputRow {
-    param(
-        [string] $Row
-    )
-    return $Row.Split("|").Trim()
-}
-
-function Create-AndroidTableObject {
-    param(
-        [string] $PackageName,
-        [string] $Description
-    )
-    return [PSCustomObject] @{
-        "Package Name" = $PackageName
-        "Description" = $Description
-    }
-}
-
 function Get-AndroidSDKManagerPath {
     return Join-Path $env:ANDROID_HOME "tools\bin\sdkmanager.bat"
 }
@@ -42,68 +31,120 @@ function Get-AndroidInstalledPackages {
     return $androidInstalledPackages
 }
 
-function Build-AndroidSDKToolsTable {
-    param (
-        [Parameter(Mandatory)]
-        [object] $packageInfo
-    )
 
-    return $packageInfo | ForEach-Object {
+function Build-AndroidTable {
-        $packageInfoParts = Split-AndroidSDKOutputRow $_
+    $packageInfo = Get-AndroidInstalledPackages
-        $packageName = $packageInfoParts[0]
+    return @(
-        $packageDescription = $packageInfoParts[2] + ", Revision " + $packageInfoParts[1]
+        @{
-        Create-AndroidTableObject -PackageName $packageName -Description $packageDescription
+            "Package" = "Android SDK Platform-Tools"
-    }
+            "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Android SDK Platform-Tools"
-}
+        },
-
+        @{
-function Build-AndroidSDKPlatformTable {
+            "Package" = "Android SDK Tools"
-    param (
+            "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Android SDK Tools"
-        [Parameter(Mandatory)]
+        },
-        [object] $packageInfo
+        @{
-    )
+            "Package" = "Android SDK Platforms"
-
+            "Version" = Get-AndroidPlatformVersions -PackageInfo $packageInfo
-    return $packageInfo | ForEach-Object {
+        },
-        $packageInfoParts = Split-AndroidSDKOutputRow $_
+        @{
-        $packageName = $packageInfoParts[0].split(";")[1]
+            "Package" = "Android SDK Build-tools"
-        $packageDescription = $packageInfoParts[2] + ", Revision " + $packageInfoParts[1]
+            "Version" = Get-AndroidBuildToolVersions -PackageInfo $packageInfo
-        return Create-AndroidTableObject -PackageName $packageName -Description $packageDescription
+        },
-    }
+        @{
-}
+            "Package" = "NDK"
-
+            "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "ndk-bundle"
-function Build-AndroidSDKBuildToolsTable {
+        },
-    param (
+        @{
-        [Parameter(Mandatory)]
+            "Package" = "Android Support Repository"
-        [object] $packageInfo
+            "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Android Support Repository"
-    )
+        },
-
+        @{
-    return $packageInfo | ForEach-Object {
+            "Package" = "Google APIs"
-        $packageInfoParts = Split-AndroidSDKOutputRow $_
+            "Version" = Get-AndroidGoogleAPIsVersions -PackageInfo $packageInfo
-        $packageName = $packageInfoParts[0].replace(";", "-")
+        },
-        $packageDescription = "Android SDK Build-Tools, Revision " + $packageInfoParts[1]
+        @{
-        return Create-AndroidTableObject -PackageName $packageName -Description $packageDescription
+            "Package" = "Google Play services"
-    }
+            "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Google Play services"
-}
+        },
-
+        @{
-function Build-AndroidExtraPackagesTable {
+            "Package" = "Google Repository"
-    param (
+            "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Google Repository"
-        [Parameter(Mandatory)][AllowEmptyString()]
+        },
-        [string[]] $installedPackages
+        @{
-    )
+            "Package" = "SDK Patch Applier v4"
-
+            "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "SDK Patch Applier v4"
-    $extraPackages = @(
+        },
-        "Android Support Repository",
+        @{
-        "Google Play services",
+            "Package" = "CMake"
-        "Google Repository",
+            "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "cmake"
-        "ndk-bundle"
+        }
-    )
+    ) | Where-Object { $_.Version } | ForEach-Object {
-
+        [PSCustomObject] @{
-    return $extraPackages | ForEach-Object {
+            "Package Name" = $_.Package
-        $packageId = $_
+            "Version" = $_.Version
-        $packageInfo = $installedPackages | Where-Object { $_ -Like "*${packageId}*" } | Select-Object -First 1
-        $packageInfoParts = Split-AndroidSDKOutputRow $packageInfo
-        return [PSCustomObject] @{
-            "Package Name" = $packageInfoParts[2]
-            "Version" = $packageInfoParts[1]
         }
     }
+}
+
+function Get-AndroidPackageVersions {
+    param (
+        [Parameter(Mandatory)]
+        [object] $PackageInfo,
+        [Parameter(Mandatory)]
+        [object] $MatchedString
+    )
+
+    $versions = $packageInfo | Where-Object { $_ -Match $MatchedString } | ForEach-Object {
+        $packageInfoParts = Split-TableRowByColumns $_
+        return $packageInfoParts[1]
+    }
+    return ($versions -Join "<br>")
+}
+
+function Get-AndroidPlatformVersions {
+    param (
+        [Parameter(Mandatory)]
+        [object] $PackageInfo
+    )
+
+    $versions = $packageInfo | Where-Object { $_ -Match "Android SDK Platform " } | ForEach-Object {
+        $packageInfoParts = Split-TableRowByColumns $_
+        $revision = $packageInfoParts[1]
+        $version = $packageInfoParts[0].split(";")[1]
+        return "$version (rev $revision)"
+    }
+    [array]::Reverse($versions)
+    return ($versions -Join "<br>")
+}
+
+function Get-AndroidBuildToolVersions {
+    param (
+        [Parameter(Mandatory)]
+        [object] $PackageInfo
+    )
+
+    $versions = $packageInfo | Where-Object { $_ -Match "Android SDK Build-Tools" } | ForEach-Object {
+        $packageInfoParts = Split-TableRowByColumns $_
+        return $packageInfoParts[1]
+    }
+    $groupVersions = @()
+    $versions | ForEach-Object {
+        $majorVersion = $_.Split(".")[0]
+        $groupVersions += $versions | Where-Object { $_.StartsWith($majorVersion) } | Join-String -Separator " "
+    }
+    return ($groupVersions | Sort-Object -Descending -Unique | Join-String -Separator "<br>")
+}
+
+function Get-AndroidGoogleAPIsVersions {
+    param (
+        [Parameter(Mandatory)]
+        [object] $PackageInfo
+    )
+
+    $versions = $packageInfo | Where-Object { $_ -Match "Google APIs" } | ForEach-Object {
+        $packageInfoParts = Split-TableRowByColumns $_
+        return $packageInfoParts[0].split(";")[1]
+    }
+    return ($versions -Join "<br>")
 }

images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1

@@ -223,32 +223,8 @@ $markdown += Get-PowerShellModules | New-MDTable
 $markdown += New-MDNewLine
 
 # Android section
-$androidInstalledPackages = Get-AndroidInstalledPackages
+$markdown += New-MDHeader "Android" -Level 3
-
+$markdown += Build-AndroidTable | New-MDTable
-$markdown += New-MDHeader "Android SDK Tools" -Level 3
-$androidSDKTools = $androidInstalledPackages | Where-Object { $_ -Match "Android SDK Tools" -or $_ -Match "Android SDK Platform-Tools" }
-$markdown += Build-AndroidSDKToolsTable $androidSDKTools | New-MDTable
 $markdown += New-MDNewLine
 
-$markdown += New-MDHeader "Android SDK Platforms" -Level 3
-$androidSDKPlatforms = $androidInstalledPackages | Where-Object { $_ -Match "Android SDK Platform " }
-$markdown += New-MDInlineCode -Text (Get-AndroidComponentLocation -ComponentName "platforms")
-$markdown += New-MDNewLine
-$markdown += Build-AndroidSDKPlatformTable $androidSDKPlatforms | New-MDTable
-$markdown += New-MDNewLine
-
-$markdown += New-MDHeader "Android SDK Build-Tools" -Level 3
-$androidSDKBuildTools = $androidInstalledPackages | Where-Object { $_ -Match "Android SDK Build-Tools" }
-$markdown += New-MDInlineCode -Text (Get-AndroidComponentLocation -ComponentName "build-tools")
-$markdown += New-MDNewLine
-$markdown += Build-AndroidSDKBuildtoolsTable $androidSDKBuildTools | New-MDTable
-$markdown += New-MDNewLine
-
-$markdown += New-MDHeader "Android Extra Packages" -Level 3
-$markdown += Build-AndroidExtraPackagesTable $androidInstalledPackages | New-MDTable
-$markdown += New-MDNewLine
-
-$markdown += New-MDHeader "Cached Docker images" -Level 3
-$markdown += New-MDList -Style Unordered -Lines @(Get-CachedDockerImages)
-
 $markdown | Out-File -FilePath "C:\InstalledSoftware.md"