[macOS] Switch image generation to faster datastores (#1801)

* select datastore script

* add debug message

* add move-vm and helpers

* add error action

* formatting the script

* nits

* add missing quotes

* add synopsis

images.CI/macos/azure-pipelines/image-generation.yml

@@ -42,6 +42,15 @@ jobs:
       SourceFolder: 'images/macos/provision/log/'
       RemoveSourceFolder: true
 
+  - task: PowerShell@2
+    displayName: 'Select datastore'
+    inputs:
+      targetType: 'filePath'
+      filePath: ./images.CI/macos/select-datastore.ps1
+      arguments: -VIServer "$(vcenter-server-v2)" `
+                 -VIUserName "$(vcenter-username-v2)" `
+                 -VIPassword "$(vcenter-password-v2)"
+
   - pwsh: |
       $SensitiveData = @(
           'IP address:',
@@ -54,7 +63,7 @@ jobs:
         -var="vcenter_password=$(vcenter-password-v2)" `
         -var="vcenter_datacenter=$(vcenter-datacenter-v2)" `
         -var="cluster_or_esxi_host=$(esxi-cluster-v2)" `
-        -var="esxi_datastore=${{ parameters.target_datastore }}" `
+        -var="esxi_datastore=$(buildDatastore)" `
         -var="output_folder=$(output-folder)" `
         -var="vm_username=$(vm-username)" `
         -var="vm_password=$(vm-password)" `
@@ -106,6 +115,18 @@ jobs:
     displayName: Publish test results
     condition: always()
 
+  - task: PowerShell@2
+    displayName: 'Move vm to cold storage'
+    condition: succeededOrFailed()
+    inputs:
+      targetType: 'filePath'
+      filePath: ./images.CI/macos/move-vm.ps1
+      arguments: -VMName "${{ variables.VirtualMachineName }}" `
+                 -TargetDataStore "${{ parameters.target_datastore }}" `
+                 -VIServer "$(vcenter-server-v2)" `
+                 -VIUserName "$(vcenter-username-v2)" `
+                 -VIPassword "$(vcenter-password-v2)"
+
   - task: PowerShell@2
     displayName: 'Destroy VM (if build canceled only)'
     condition: eq(variables['Agent.JobStatus'], 'Canceled')

images.CI/macos/destroy-vm.ps1

@@ -1,3 +1,21 @@
+<#
+.SYNOPSIS
+
+This script deletes vm from vCenter
+
+.PARAMETER VMName
+VM name to delete (Example "macOS-10.15_20201012.4")
+
+.PARAMETER VIServer
+vCenter address (Example "10.0.1.16")
+
+.PARAMETER VIUserName
+vCenter username (Example "Administrator")
+
+.PARAMETER VIPassword
+vCenter password (Example "12345678")
+#>
+
 [CmdletBinding()]
 param(
     [Parameter(Mandatory)]
@@ -17,25 +35,13 @@ param(
     [string]$VIPassword
 )
 
-$ProgressPreference = "SilentlyContinue"
-$WarningPreference = "SilentlyContinue"
+# Import helpers module
+Import-Module $PSScriptRoot\helpers.psm1 -DisableNameChecking
 
-# connection to a vCenter Server system
-try
-{
-    $null = Set-PowerCLIConfiguration -Scope Session -InvalidCertificateAction Ignore -ParticipateInCEIP $false -Confirm:$false -WebOperationTimeoutSeconds 600
-    $securePassword = ConvertTo-SecureString -String $VIPassword -AsPlainText -Force
-    $cred = New-Object System.Management.Automation.PSCredential($VIUserName, $securePassword)
-    $null = Connect-VIServer -Server $VIServer -Credential $cred -ErrorAction Stop
-    Write-Host "Connection to the vSphere server has been established"
-}
-catch
-{
-    Write-Host "##vso[task.LogIssue type=error;]Failed to connect to the vSphere server"
-    exit 1
-}
+# Connection to a vCenter Server system
+Connect-VCServer
 
-# check vm clone status
+# Check vm clone status
 $chainId = (Get-VIEvent -Entity $VMName).ChainId
 if ($chainId)
 {
@@ -45,7 +51,7 @@ if ($chainId)
         try
         {
             Stop-Task -Task $task -Confirm:$false -ErrorAction Stop
-            Write-Host "The vm '$VMName' clone task has been cancelled"
+            Write-Host "The vm '$VMName' clone task has been canceled"
         }
         catch
         {
@@ -54,7 +60,7 @@ if ($chainId)
     }
 }
 
-# remove a vm
+# Remove a vm
 $vm = Get-VM -Name $VMName -ErrorAction SilentlyContinue
 
 if ($vm)

images.CI/macos/helpers.psm1

@@ -0,0 +1,26 @@
+<#
+.SYNOPSIS
+
+Helper functions to use in images.CI scripts
+#>
+
+Function Connect-VCServer
+{
+    try
+    {
+        # Preference
+        $global:ProgressPreference = 'SilentlyContinue'
+        $global:WarningPreference = 'SilentlyContinue'
+        # Ignore SSL
+        $null = Set-PowerCLIConfiguration -Scope Session -InvalidCertificateAction Ignore -ParticipateInCEIP $false -Confirm:$false -WebOperationTimeoutSeconds 600
+        $securePassword = ConvertTo-SecureString -String $VIPassword -AsPlainText -Force
+        $cred = New-Object System.Management.Automation.PSCredential($VIUserName, $securePassword)
+        $null = Connect-VIServer -Server $VIServer -Credential $cred -ErrorAction Stop
+        Write-Host "Connection to the vSphere server has been established"
+    }
+    catch
+    {
+        Write-Host "##vso[task.LogIssue type=error;]Failed to connect to the vSphere server"
+        exit 1
+    }
+}

images.CI/macos/move-vm.ps1

@@ -0,0 +1,59 @@
+<#
+.SYNOPSIS
+
+This script migrates given VM to another datastore
+
+.PARAMETER VMName
+VM name to migrate (Example "macOS-10.15_20201012.4")
+
+.PARAMETER TargetDataStore
+Target datastore (Example "ds-image")
+
+.PARAMETER VIServer
+vCenter address (Example "10.0.1.16")
+
+.PARAMETER VIUserName
+vCenter username (Example "Administrator")
+
+.PARAMETER VIPassword
+vCenter password (Example "12345678")
+#>
+
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory)]
+    [ValidateNotNullOrEmpty()]
+    [string]$VMName,
+
+    [Parameter(Mandatory)]
+    [ValidateNotNullOrEmpty()]
+    [string]$TargetDataStore,
+
+    [Parameter(Mandatory)]
+    [ValidateNotNullOrEmpty()]
+    [string]$VIServer,
+
+    [Parameter(Mandatory)]
+    [ValidateNotNullOrEmpty()]
+    [string]$VIUserName,
+
+    [Parameter(Mandatory)]
+    [ValidateNotNullOrEmpty()]
+    [string]$VIPassword
+)
+
+# Import helpers module
+Import-Module $PSScriptRoot\helpers.psm1 -DisableNameChecking
+
+# Connection to a vCenter Server system
+Connect-VCServer
+
+try
+{
+    Get-VM $VMName | Move-VM -Datastore $TargetDataStore -ErrorAction Stop
+    Write-Host "VM has been moved successfully to target datastore '$TargetDataStore'"
+}
+catch
+{
+    Write-Host "##vso[task.LogIssue type=error;]Failed to move VM '$VMName' to target datastore '$TargetDataStore'"
+}

images.CI/macos/select-datastore.ps1

@@ -0,0 +1,64 @@
+<#
+.SYNOPSIS
+
+This script selects local datastore based on the following rules:
+
+- Name starts with ds-local-Datastore
+- Datastore FreespaceGB > 400 Gb
+- VM count on the datastore < 2
+
+.PARAMETER VIServer
+vCenter address (Example "10.0.1.16")
+
+.PARAMETER VIUserName
+vCenter username (Example "Administrator")
+
+.PARAMETER VIPassword
+vCenter password (Example "12345678")
+#>
+
+
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory)]
+    [ValidateNotNullOrEmpty()]
+    [string]$VIServer,
+
+    [Parameter(Mandatory)]
+    [ValidateNotNullOrEmpty()]
+    [string]$VIUserName,
+
+    [Parameter(Mandatory)]
+    [ValidateNotNullOrEmpty()]
+    [string]$VIPassword
+)
+
+# Import helpers module
+Import-Module $PSScriptRoot\helpers.psm1 -DisableNameChecking
+
+# Connection to a vCenter Server system
+Connect-VCServer
+
+# Get a target datastore for current deployment
+# 1. Name starts with ds-local-Datastore
+# 2. FreespaceGB > 400 Gb
+# 3. VM count on a datastore < 2
+$templateDatastore = "ds-local-Datastore-*"
+$thresholdInGb = 400
+$vmCount = 2
+$allDatastores = Get-Datastore -Name $templateDatastore | Where-Object { $_.State -eq "Available" }
+$buildDatastore = $allDatastores | Where-Object { $_.FreeSpaceGB -ge $thresholdInGb } | Where-Object {
+            $vmOnDatastore = @((Get-ChildItem -Path $_.DatastoreBrowserPath).Name -notmatch "^\.").Count
+            $vmOnDatastore -lt $vmCount
+        } | Select-Object -ExpandProperty Name -First 1
+
+if ($buildDatastore)
+{
+    Write-Host "Datastore selected successfully"
+    Write-Host "##vso[task.setvariable variable=buildDatastore;issecret=true]$buildDatastore"
+}
+else
+{
+    Write-Host "##vso[task.LogIssue type=error;]No datastores found for the condition"
+    exit 1
+}