Implement public CI for MacOS builds

images.CI/linux-and-win/azure-pipelines/image-generation.yml

@@ -20,13 +20,13 @@ jobs:
       targetType: 'filePath'
       filePath: ./images.CI/download-repo.ps1
       arguments: -RepoUrl $(CUSTOM_REPOSITORY_URL) `
-                        -RepoBranch $(CUSTOM_REPOSITORY_BRANCH)
+                 -RepoBranch $(CUSTOM_REPOSITORY_BRANCH)
 
   - task: PowerShell@2
     displayName: 'Build VM'
     inputs:
       targetType: filePath
-      filePath: ./images.CI/build-image.ps1
+      filePath: ./images.CI/linux-and-win/build-image.ps1
       arguments: -ResourcesNamePrefix $(Build.BuildId) `
                         -ClientId $(CLIENT_ID) `
                         -ClientSecret $(CLIENT_SECRET) `
@@ -45,7 +45,7 @@ jobs:
     displayName: 'Create release for VM deployment'
     inputs:
       targetType: filePath
-      filePath: ./images.CI/create-release.ps1
+      filePath: ./images.CI/linux-and-win/create-release.ps1
       arguments: -BuildId $(Build.BuildId) `
                         -Organization $(RELEASE_TARGET_ORGANIZATION) `
                         -DefinitionId $(RELEASE_TARGET_DEFINITION_ID) `
@@ -66,7 +66,7 @@ jobs:
     condition: always()
     inputs:
       targetType: filePath
-      filePath: ./images.CI/cleanup.ps1
+      filePath: ./images.CI/linux-and-win/cleanup.ps1
       arguments: -ResourcesNamePrefix $(Build.BuildId) `
                      -ClientId $(CLIENT_ID) `
                      -ClientSecret $(CLIENT_SECRET) `

images.CI/macos/azure-pipelines/image-generation.yml

@@ -0,0 +1,105 @@
+jobs:
+- job: Image_generation
+  displayName: Image Generation (${{ parameters.image_label }})
+  timeoutInMinutes: 720
+  pool:
+    name: Mac-Cloud Image Generation
+  variables:
+  - group: Mac-Cloud Image Generation
+  - group: Mac-Cloud Image Generation Key Vault
+
+  steps:
+  - checkout: self
+    clean: true
+    fetchDepth: 1
+
+  - task: PowerShell@2
+    displayName: 'Validate contributor permissions'
+    condition: startsWith(variables['Build.SourceBranch'], 'refs/pull/')
+    inputs:
+      targetType: 'filePath'
+      filePath: ./images.CI/macos/validate-contributor.ps1
+      pwsh: true
+      arguments: -RepositoryName "$(Build.Repository.Name)" `
+                 -AccessToken "$(github-feed-token)" `
+                 -SourceBranch "$(Build.SourceBranch)" `
+                 -ContributorAllowList "$(CONTRIBUTOR_ALLOWLIST)"
+
+  - task: PowerShell@2
+    displayName: 'Download custom repository'
+    condition: and(ne(variables['CUSTOM_REPOSITORY_URL'], ''), ne(variables['CUSTOM_REPOSITORY_BRANCH'], ''))
+    inputs:
+      targetType: 'filePath'
+      filePath: ./images.CI/download-repo.ps1
+      arguments: -RepoUrl $(CUSTOM_REPOSITORY_URL) `
+                 -RepoBranch $(CUSTOM_REPOSITORY_BRANCH)
+
+  - task: DeleteFiles@1
+    displayName: Clean up self-hosted machine
+    inputs:
+      SourceFolder: 'images/macos/provision/log/'
+      RemoveSourceFolder: true
+
+  - pwsh: |
+      $SensitiveData = @(
+          'IP address:',
+          'Using ssh communicator to connect:'
+      )
+
+      packer build -on-error=abort `
+        -var="vcenter_server=$(vcenter-server-v2)" `
+        -var="vcenter_username=$(vcenter-username-v2)" `
+        -var="vcenter_password=$(vcenter-password-v2)" `
+        -var="vcenter_datacenter=$(vcenter-datacenter-v2)" `
+        -var="cluster_or_esxi_host=$(esxi-cluster-v2)" `
+        -var="esxi_datastore=${{ parameters.target_datastore }}" `
+        -var="output_folder=$(output-folder)" `
+        -var="vm_username=$(vm-username)" `
+        -var="vm_password=$(vm-password)" `
+        -var="build_id=$(Build.BuildNumber)" `
+        -var="baseimage_name=${{ parameters.base_image_name }}" `
+        -var="github_feed_token=$(github-feed-token)" `
+        -var="xcode_install_user=$(xcode-installation-user)" `
+        -var="xcode_install_password=$(xcode-installation-password)" `
+        -color=false `
+        ${{ parameters.template_path }} `
+        | Where-Object {
+            #Filter sensitive data from Packer logs
+            $currentString = $_
+            $sensitiveString = $SensitiveData | Where-Object { $currentString -match $_ }
+            $sensitiveString -eq $null
+        }
+    displayName: 'Build VM'
+    env:
+      PACKER_LOG: 0
+    workingDirectory: 'images/macos'
+
+  - bash: |
+      echo "Copy image output files"
+      cp -R "images/image-output/software-report/." "$(Build.ArtifactStagingDirectory)/"
+
+      echo "Copy test results"
+      cp -R "images/image-output/tests/." "$(Common.TestResultsDirectory)/"
+      ls $(Common.TestResultsDirectory)
+
+      echo "Put VM name to 'VM_Done_Name' file"
+      echo "$(Build.BuildNumber)" > "$(Build.ArtifactStagingDirectory)/VM_Done_Name"
+    displayName: Prepare artifact
+
+  - bash: |
+      cat "$(Build.ArtifactStagingDirectory)/systeminfo.md"
+    displayName: Print software report
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      ArtifactName: 'Built_VM_Artifacts'
+    displayName: Publish Artifacts
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '*.xml'
+      testResultsFormat: NUnit
+      searchFolder: '$(Common.TestResultsDirectory)'
+      failTaskOnFailedTests: true
+    displayName: Publish test results
+    condition: always()

images.CI/macos/azure-pipelines/macos1013.yml

@@ -0,0 +1,23 @@
+name: macOS-10.13_$(date:yyyyMMdd)$(rev:.r)_unstable
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily
+  branches:
+    include:
+    - main
+  always: true
+
+trigger: none
+pr:
+  autoCancel: true
+  branches:
+    include:
+    - main
+
+jobs:
+- template: image-generation.yml
+  parameters:
+    image_label: 'macOS High Sierra'
+    base_image_name: 'clean-macOS-10.13.6-380Gb-SIPoff_runner'
+    template_path: 'templates/macOS-10.13.json'
+    target_datastore: 'ds-image'

images.CI/macos/azure-pipelines/macos1014.yml

@@ -0,0 +1,23 @@
+name: macOS-10.14_$(date:yyyyMMdd)$(rev:.r)_unstable
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily
+  branches:
+    include:
+    - main
+  always: true
+
+trigger: none
+pr:
+  autoCancel: true
+  branches:
+    include:
+    - main
+
+jobs:
+- template: image-generation.yml
+  parameters:
+    image_label: 'macOS Mojave'
+    base_image_name: 'clean-macOS-10.14-380Gb_runner'
+    template_path: 'templates/macOS-10.14.json'
+    target_datastore: 'ds-image'

images.CI/macos/azure-pipelines/macos1015.yml

@@ -0,0 +1,23 @@
+name: macOS-10.15_$(date:yyyyMMdd)$(rev:.r)_unstable
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily
+  branches:
+    include:
+    - main
+  always: true
+
+trigger: none
+pr:
+  autoCancel: true
+  branches:
+    include:
+    - main
+
+jobs:
+- template: image-generation.yml
+  parameters:
+    image_label: 'macOS Catalina'
+    base_image_name: 'clean-macOS-10.15-380Gb-runner'
+    template_path: 'templates/macOS-10.15.json'
+    target_datastore: 'ds-image'

images.CI/macos/azure-pipelines/macos110.yml

@@ -0,0 +1,23 @@
+name: macOS-11.0_$(date:yyyyMMdd)$(rev:.r)_unstable
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily
+  branches:
+    include:
+    - main
+  always: true
+
+trigger: none
+pr:
+  autoCancel: true
+  branches:
+    include:
+    - main
+
+jobs:
+- template: image-generation.yml
+  parameters:
+    image_label: 'macOS Big Sur'
+    base_image_name: 'clean-macOS-11.0-380Gb-runner'
+    template_path: 'templates/macOS-11.0.json'
+    target_datastore: 'ds-image'

images.CI/macos/validate-contributor.ps1

@@ -0,0 +1,63 @@
+param(
+    [Parameter(Mandatory)] [string] $RepositoryName,
+    [Parameter(Mandatory)] [string] $AccessToken,
+    [Parameter(Mandatory)] [string] $SourceBranch,
+    [Parameter(Mandatory)] [string] $ContributorAllowList
+)
+
+function Build-AuthHeader {
+    param(
+        [Parameter(Mandatory)] [string] $AccessToken
+    )
+
+    $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("'':${AccessToken}"))
+    return "Basic ${base64AuthInfo}"
+}
+
+function Get-PullRequest {
+    param(
+        [Parameter(Mandatory)] [string] $RepositoryName,
+        [Parameter(Mandatory)] [string] $AccessToken,
+        [Parameter(Mandatory)] [UInt32] $PullRequestNumber
+    )
+
+    $requestUrl = "https://api.github.com/repos/$RepositoryName/pulls/$PullRequestNumber"
+    $authHeader = Build-AuthHeader -AccessToken $AccessToken
+
+    $params = @{
+        Method = "GET"
+        ContentType = "application/json"
+        Uri = $requestUrl
+        Headers = @{ Authorization = $authHeader }
+    }
+
+    return Invoke-RestMethod @params
+}
+
+function Validate-ContributorPermissions {
+    param(
+        [Parameter(Mandatory)] [string] $ContributorAllowList,
+        [Parameter(Mandatory)] [string] $ContributorName
+    )
+
+    $allowedContributors = $ContributorAllowList.Split(",").Trim()
+    $validСontributor = $allowedContributors | Where-Object { $_ -eq $ContributorName } `
+                                             | Select-Object -First 1
+
+    if (-not $validСontributor) {
+       Write-Host "Failed to start this build. '$ContributorName' is an unknown contributor"
+       Write-Host "Please add '$ContributorName' to the allowed list to run builds"
+       exit 1
+    }
+}
+
+$pullRequestNumber = $SourceBranch.Split("/")[2]
+
+$pullRequestInfo = Get-PullRequest -RepositoryName $RepositoryName `
+                                   -AccessToken $AccessToken `
+                                   -PullRequestNumber $pullRequestNumber
+
+$contributorName = $pullRequestInfo.user.login
+
+Validate-ContributorPermissions -ContributorAllowList $ContributorAllowList `
+                                -ContributorName $contributorName