Add GitHub actions for image generation (#7182)

.github/workflows/macos-generation.yml

@@ -0,0 +1,212 @@
+name: macOS image generation
+on:
+  workflow_call:
+    inputs:
+      image_label:
+        type: string
+        description: macOS codename
+        required: true
+      base_image_name:
+        type: string
+        description: Base clean image
+        required: true
+      template_path:
+        type: string
+        description: Packer template path
+        required: true
+      target_datastore:
+        type: string
+        description: Image datastore
+        required: true
+      custom_repo:
+        type: string
+        description: Custom repo to checkout
+        required: false
+      custom_repo_commit_hash:
+        type: string
+        description: Custom repo commit hash
+        required: false
+
+env:
+  KEYVAULT: imagegeneration
+  ESXI_CLUSTER: mcv2-build-unstable
+  VCENTER_DATACENTER: imagegen
+  OUTPUT_FOLDER: mms-output
+  BUILD_DATASTORE: ds-image
+
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  build:
+    runs-on: macos-vmware
+    timeout-minutes: 1200
+    steps:
+    - uses: azure/login@v1
+      with:
+        creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+    - name: Set image variables
+      run: |
+        $currentDate = Get-Date -Format "yyyyMMdd"
+        $templatePath = "${{ inputs.template_path }}"
+        $osName = $(($templatePath.Split("/")[-1]).Split(".")[0])
+        $virtualMachineName = "${osName}_${currentDate}_unstable.${{ github.run_id }}.${{ github.run_attempt }}"
+        $GitHubFeed = az keyvault secret show -n "github-feed-token" --vault-name "${{ env.KEYVAULT }}" --query value -o tsv
+        $VIUserName = az keyvault secret show -n "vcenter-username-v2" --vault-name "${{ env.KEYVAULT }}" --query value -o tsv
+        $VIPassword = az keyvault secret show -n "vcenter-password-v2" --vault-name "${{ env.KEYVAULT }}" --query value -o tsv
+        $XcodeUser = az keyvault secret show -n "xcode-installation-user" --vault-name "${{ env.KEYVAULT }}" --query value -o tsv
+        $XcodePassword =  az keyvault secret show -n "xcode-installation-password" --vault-name "${{ env.KEYVAULT }}" --query value -o tsv
+        echo "::add-mask::$GitHubFeed"
+        echo "::add-mask::$VIUserName"
+        echo "::add-mask::$VIPassword"
+        echo "::add-mask::$XcodeUser"
+        echo "::add-mask::$XcodePassword"
+        "GH_FEED=$GitHubFeed" | Out-File -Append -FilePath $env:GITHUB_ENV
+        "VI_USER_NAME=$VIUserName" | Out-File -Append -FilePath $env:GITHUB_ENV
+        "VI_PASSWORD=$VIPassword" | Out-File -Append -FilePath $env:GITHUB_ENV
+        "XCODE_USER=$XcodeUser" | Out-File -Append -FilePath $env:GITHUB_ENV
+        "XCODE_PASSWORD=$XcodePassword" | Out-File -Append -FilePath $env:GITHUB_ENV
+        "VM_NAME=$virtualMachineName" | Out-File -Append -FilePath $env:GITHUB_ENV
+
+    - name: Determine checkout type
+      run: |
+        if ("${{ inputs.custom_repo }}" -and "${{ inputs.custom_repo_commit_hash }}") {
+            $checkoutType = "custom_repo"
+        } elseif (("${{ github.event_name }}" -eq "pull_request_target") -and ("${{ github.event.action }}" -eq "labeled" )) {
+            $checkoutType = "pull_request"
+        } else {
+            $checkoutType = "main"
+        }
+        "CHECKOUT_TYPE=$checkoutType" | Out-File -Append $env:GITHUB_ENV
+
+    - name: Checkout repository
+      if: ${{ env.CHECKOUT_TYPE == 'main' }}
+      uses: actions/checkout@v3
+      with:
+        repository: actions/runner-images
+
+    - name: Checkout PR
+      if: ${{ env.CHECKOUT_TYPE == 'pull_request' }}
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: Checkout custom repository
+      if: ${{ env.CHECKOUT_TYPE == 'custom_repo' }}
+      uses: actions/checkout@v3
+      with:
+        repository: '${{ inputs.custom_repo }}'
+        ref: '${{ inputs.custom_repo_commit_hash }}'
+
+    - name: Validate contributor permissions
+      if: ${{ github.event_name == 'pull_request_target' }}
+      run: |
+        
+        [string]$contributorAllowList = "${{ vars.CONTRIBUTOR_ALLOWLIST }}"
+        ./images.CI/macos/validate-contributor.ps1 `
+          -RepositoryName ${{ github.repository }} `
+          -AccessToken ${{ env.GH_FEED }} `
+          -SourceBranch "refs/pull/${{ github.event.pull_request.number }}/merge" `
+          -ContributorAllowList $contributorAllowList
+
+    - name: Select datastore
+      run: |
+        ./images.CI/macos/select-datastore.ps1 `
+          -VMName "${{ env.VM_NAME }}" `
+          -VIServer ${{ secrets.VISERVER_V2 }} `
+          -VIUserName ${{ env.VI_USER_NAME }} `
+          -VIPassword ${{ env.VI_PASSWORD }} `
+          -Cluster ${{ env.ESXI_CLUSTER }}
+
+    - name: Build VM
+      run: |
+        $SensitiveData = @(
+          'IP address:',
+          'Using ssh communicator to connect:'
+        )
+         packer build -on-error=abort `
+        -var="vcenter_server=${{ secrets.VISERVER_V2 }}" `
+        -var="vcenter_username=${{ env.VI_USER_NAME }}" `
+        -var="vcenter_password=${{ env.VI_PASSWORD }}" `
+        -var="vcenter_datacenter=${{ env.VCENTER_DATACENTER }}" `
+        -var="cluster_or_esxi_host=${{ env.ESXI_CLUSTER }}" `
+        -var="esxi_datastore=${{ env.BUILD_DATASTORE }}" `
+        -var="output_folder=${{ env.OUTPUT_FOLDER }}" `
+        -var="vm_username=${{ secrets.VM_USERNAME }}" `
+        -var="vm_password=${{ secrets.VM_PASSWORD }}" `
+        -var="github_api_pat=${{ secrets.GH_FEED_TOKEN }}" `
+        -var="build_id=${{ env.VM_NAME }}" `
+        -var="baseimage_name=${{ inputs.base_image_name }}" `
+        -var="xcode_install_user=${{ env.XCODE_USER }}" `
+        -var="xcode_install_password=${{ env.XCODE_PASSWORD }}" `
+        -color=false `
+        ${{ inputs.template_path }} `
+        | Where-Object {
+            #Filter sensitive data from Packer logs
+            $currentString = $_
+            $sensitiveString = $SensitiveData | Where-Object { $currentString -match $_ }
+            $sensitiveString -eq $null
+        }
+      working-directory: images/macos
+      env:
+        PACKER_LOG: 1
+        PACKER_LOG_PATH: ${{ runner.temp }}/packer-log.txt
+
+    - name: Prepare artifact
+      shell: bash
+      run: |
+        echo "Preparing artifact directory"
+        mkdir -p ${{ runner.temp }}/artifacts
+
+        echo "Copy image output files"
+        cp -R "images/image-output/software-report/." "${{ runner.temp }}/artifacts"
+
+        echo "Put VM name to 'VM_Done_Name' file"
+        echo "${{ env.VM_NAME }}" > "${{ runner.temp }}/artifacts/VM_Done_Name"
+
+    - name: Print markdown software report
+      run: |
+        Get-Content "${{ runner.temp }}/artifacts/systeminfo.md"
+
+    - name: Print json software report
+      run: |
+        Get-Content "${{ runner.temp }}/artifacts/systeminfo.json"
+
+    - name: Publish Artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: Built_VM_Artifacts
+        path: ${{ runner.temp }}/artifacts/
+
+    - name: Print provisioners duration
+      run: |
+        ./images.CI/measure-provisioners-duration.ps1 `
+          -PackerLogPath "${{ runner.temp }}/packer-log.txt" `
+          -PrintTopNLongest 25
+
+    - name: Move vm to cold storage and clear datastore tag
+      run: |
+        $cpuCount = 3
+        $coresPerSocketCount = 3
+        $memory = 14336
+
+        ./images.CI/macos/move-vm.ps1 `
+          -VMName "${{ env.VM_NAME }}" `
+          -TargetDataStore "${{ inputs.target_datastore }}" `
+          -VIServer "${{ secrets.VISERVER_V2 }}" `
+          -VIUserName "${{ env.VI_USER_NAME }}" `
+          -VIPassword "${{ env.VI_PASSWORD }}" `
+          -CpuCount "$cpuCount" `
+          -CoresPerSocketCount "$coresPerSocketCount" `
+          -Memory "$memory"
+
+    - name: Destroy VM (if build canceled only)
+      if: ${{ cancelled() }}
+      run: |
+        ./images.CI/macos/destroy-vm.ps1 `
+          -VMName "${{ env.VM_NAME }}" `
+          -VIServer "${{ secrets.VISERVER_V2 }}" `
+          -VIUserName "${{ env.VI_USER_NAME }}" `
+          -VIPassword "${{ env.VI_PASSWORD }}"

.github/workflows/macos11.yml

@@ -0,0 +1,30 @@
+run-name: macOS-11_unstable.${{ github.run_id }}.${{ github.run_attempt }}
+on:
+  workflow_dispatch:
+    inputs:
+      CUSTOM_REPOSITORY:
+        description: 'Custom repository (owner/repo)'
+        required: false
+      CUSTOM_REPOSITORY_COMMIT_HASH:
+        description: 'Commit hash'
+        required: false
+  pull_request_target:
+    types: labeled
+    paths:
+    - 'images/macos/**'
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  macOS_11:
+    if: ${{ (github.event.label.name == 'macos-all') || (github.event.label.name == 'macos-11') || (inputs.CUSTOM_REPOSITORY != '' && inputs.CUSTOM_REPOSITORY_COMMIT_HASH != '') || (github.event_name == 'schedule') }}
+    name: macOS-11_unstable.${{ github.run_id }}.${{ github.run_attempt }}
+    uses: ./.github/workflows/macos-generation.yml
+    with:
+      image_label: 'macOS Big Sur'
+      base_image_name: 'clean-macOS-11-380Gb-runner'
+      template_path: 'templates/macOS-11.json'
+      target_datastore: 'ds-image'
+      custom_repo: ${{ github.event.inputs.CUSTOM_REPOSITORY }}
+      custom_repo_hash: ${{ github.event.inputs.CUSTOM_REPOSITORY_COMMIT_HASH }}
+    secrets: inherit

.github/workflows/macos12.yml

@@ -0,0 +1,30 @@
+run-name: macOS-12_unstable.${{ github.run_id }}.${{ github.run_attempt }}
+on:
+  workflow_dispatch:
+    inputs:
+      CUSTOM_REPOSITORY:
+        description: 'Custom repository (owner/repo)'
+        required: false
+      CUSTOM_REPOSITORY_COMMIT_HASH:
+        description: 'Commit hash'
+        required: false
+  pull_request_target:
+    types: labeled
+    paths:
+    - 'images/macos/**'
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  macOS_12:
+    if: ${{ (github.event.label.name == 'macos-all') || (github.event.label.name == 'macos-12') || (inputs.CUSTOM_REPOSITORY != '' && inputs.CUSTOM_REPOSITORY_COMMIT_HASH != '') || (github.event_name == 'schedule') }}
+    name: macOS-12_unstable.${{ github.run_id }}.${{ github.run_attempt }}
+    uses: ./.github/workflows/macos-generation.yml
+    with:
+      image_label: 'macOS Monterey'
+      base_image_name: 'clean-macOS-12-380Gb-runner'
+      template_path: 'templates/macOS-12.json'
+      target_datastore: 'ds-image'
+      custom_repo: ${{ github.event.inputs.CUSTOM_REPOSITORY }}
+      custom_repo_commit_hash: ${{ github.event.inputs.CUSTOM_REPOSITORY_COMMIT_HASH }}
+    secrets: inherit

.github/workflows/ubuntu-win-generation.yml

@@ -0,0 +1,128 @@
+name: MMS image generation
+on:
+  workflow_call:
+    inputs:
+      image_name:
+        type: string
+        description: An OS image to build
+        required: true
+      image_readme_name:
+        type: string
+        description: README file path
+        required: true
+      custom_repo:
+        type: string
+        description: Custom repo to checkout
+        required: false
+      custom_repo_commit_hash:
+        type: string
+        description: Custom repo commit hash
+        required: false
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  build:
+    runs-on: azure-builds
+    timeout-minutes: 1200
+    steps:
+      - name: Determine checkout type
+        run: |
+          if ("${{ inputs.custom_repo }}" -and "${{ inputs.custom_repo_commit_hash }}") {
+              $checkoutType = "custom_repo"
+          } elseif (("${{ github.event_name }}" -eq "pull_request_target") -and ("${{ github.event.action }}" -eq "labeled" )) {
+              $checkoutType = "pull_request"
+          } else {
+              $checkoutType = "main"
+          }
+          "CHECKOUT_TYPE=$checkoutType" | Out-File -Append $env:GITHUB_ENV
+
+      - name: Checkout repository
+        if: ${{ env.CHECKOUT_TYPE == 'main' }}
+        uses: actions/checkout@v3
+        with:
+          repository: actions/runner-images
+
+      - name: Checkout PR
+        if: ${{ env.CHECKOUT_TYPE == 'pull_request' }}
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Checkout custom repository
+        if: ${{ env.CHECKOUT_TYPE == 'custom_repo' }}
+        uses: actions/checkout@v3
+        with:
+          repository: '${{ inputs.custom_repo }}'
+          ref: '${{ inputs.custom_repo_commit_hash }}'
+
+      - name: Set image varibles
+        run: |
+          $ImageType = "${{ inputs.image_name }}"
+
+          if ($ImageType.StartsWith("ubuntu")) { $TemplateDirectoryName = "linux" } else { $TemplateDirectoryName = "win" }
+
+          $TemplateDirectoryPath = Join-Path "images" $TemplateDirectoryName | Resolve-Path
+          $TemplatePath = Join-Path $TemplateDirectoryPath "$ImageType.pkr.hcl"
+
+          if ( -not (Test-Path $TemplatePath) ) {
+            $TemplatePath = Join-Path $TemplateDirectoryPath "$ImageType.json"
+          }
+
+          "TemplatePath=$TemplatePath" | Out-File -Append -FilePath $env:GITHUB_ENV
+          "TemplateDirectoryPath=$TemplateDirectoryPath" | Out-File -Append -FilePath $env:GITHUB_ENV
+          "ImageType=$ImageType" | Out-File -Append -FilePath $env:GITHUB_ENV
+
+      - name: Build image
+        run: |
+            ./images.CI/linux-and-win/build-image.ps1 `
+              -TemplatePath ${{ env.TemplatePath }} `
+              -ClientId ${{ secrets.CLIENT_ID }} `
+              -ClientSecret ${{ secrets.CLIENT_SECRET }} `
+              -Location ${{ secrets.AZURE_LOCATION }} `
+              -ResourcesNamePrefix ${{ github.run_number }} `
+              -ResourceGroup ${{ secrets.AZURE_RESOURCE_GROUP }} `
+              -StorageAccount ${{ secrets.AZURE_STORAGE_ACCOUNT }} `
+              -SubscriptionId ${{ secrets.AZURE_SUBSCRIPTION }} `
+              -TenantId ${{ secrets.AZURE_TENANT }} `
+              -VirtualNetworkName ${{ secrets.BUILD_AGENT_VNET_NAME }} `
+              -VirtualNetworkSubnet ${{ secrets.BUILD_AGENT_SUBNET_NAME }} `
+              -VirtualNetworkRG ${{ secrets.BUILD_AGENT_VNET_RESOURCE_GROUP }} `
+        env:
+          PACKER_LOG: 1
+          PACKER_LOG_PATH: ${{ runner.temp }}/packer-log.txt
+          RUN_VALIDATION_FLAG: true
+
+      - name: Output Readme file content
+        run: |
+          Get-Content -Path (Join-Path "$env:TemplateDirectoryPath" "${{ inputs.image_readme_name }}")
+
+      - name: Print provisioners duration
+        run: |
+          ./images.CI/measure-provisioners-duration.ps1 `
+            -PackerLogPath "${{ runner.temp }}/packer-log.txt" `
+            -PrefixToPathTrim ${{ env.TemplateDirectoryPath }} `
+            -PrintTopNLongest 25
+
+      - name: Create release for VM deployment
+        run: |
+          ./images.CI/linux-and-win/create-release.ps1 `
+            -BuildId ${{ github.run_number }} `
+            -Organization ${{ secrets.RELEASE_TARGET_ORGANIZATION }} `
+            -DefinitionId ${{ secrets.RELEASE_TARGET_DEFINITION_ID }} `
+            -Project ${{ secrets.RELEASE_TARGET_PROJECT }} `
+            -ImageName ${{ env.ImageType }} `
+            -AccessToken ${{ secrets.RELEASE_TARGET_TOKEN }}
+
+      - name: Clean up resources
+        if: ${{ always() }}
+        run: |
+          ./images.CI/linux-and-win/cleanup.ps1 `
+            -ResourcesNamePrefix ${{ github.run_number }} `
+            -Image ${{ env.ImageType }} `
+            -StorageAccount ${{ secrets.AZURE_STORAGE_ACCOUNT }} `
+            -SubscriptionId ${{ secrets.AZURE_SUBSCRIPTION }} `
+            -ClientId ${{ secrets.CLIENT_ID }} `
+            -ClientSecret ${{ secrets.CLIENT_SECRET }} `
+            -TenantId ${{ secrets.AZURE_TENANT }}

.github/workflows/ubuntu2004.yml

@@ -0,0 +1,27 @@
+run-name: Ubuntu20.04 - ${{ (github.event.pull_request.title || 'scheduled/manual run') }}
+on:
+  workflow_dispatch:
+    inputs:
+      CUSTOM_REPOSITORY:
+        description: 'Custom repository (owner/repo)'
+        required: false
+      CUSTOM_REPOSITORY_COMMIT_HASH:
+        description: 'Commit hash'
+        required: false
+  pull_request_target:
+    types: labeled
+    paths:
+    - 'images/linux/**'
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  Ubuntu_2004:
+    if: ${{ (github.event.label.name == 'ubuntu-all') || (github.event.label.name == 'ubuntu-2004') || (inputs.CUSTOM_REPOSITORY != '' && inputs.CUSTOM_REPOSITORY_COMMIT_HASH != '') || (github.event_name == 'schedule') }}
+    uses: ./.github/workflows/ubuntu-win-generation.yml
+    with:
+      image_name: 'ubuntu2004'
+      image_readme_name: 'Ubuntu2004-Readme.md'
+      custom_repo: ${{ github.event.inputs.CUSTOM_REPOSITORY }}
+      custom_repo_commit_hash: ${{ github.event.inputs.CUSTOM_REPOSITORY_COMMIT_HASH }}
+    secrets: inherit

.github/workflows/ubuntu2204.yml

@@ -0,0 +1,27 @@
+run-name: Ubuntu22.04 - ${{ (github.event.pull_request.title || 'scheduled/manual run') }}
+on:
+  workflow_dispatch:
+    inputs:
+      CUSTOM_REPOSITORY:
+        description: 'Custom repository (owner/repo)'
+        required: false
+      CUSTOM_REPOSITORY_COMMIT_HASH:
+        description: 'Commit hash'
+        required: false
+  pull_request_target:
+    types: labeled
+    paths:
+    - 'images/linux/**'
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  Ubuntu_2204:
+    if: ${{ (github.event.label.name == 'ubuntu-all') || (github.event.label.name == 'ubuntu-2204') || (inputs.CUSTOM_REPOSITORY != '' && inputs.CUSTOM_REPOSITORY_COMMIT_HASH != '') || (github.event_name == 'schedule') }}
+    uses: ./.github/workflows/ubuntu-win-generation.yml
+    with:
+      image_name: 'ubuntu2204'
+      image_readme_name: 'Ubuntu2204-Readme.md'
+      custom_repo: ${{ github.event.inputs.CUSTOM_REPOSITORY }}
+      custom_repo_commit_hash: ${{ github.event.inputs.CUSTOM_REPOSITORY_COMMIT_HASH }}
+    secrets: inherit

.github/workflows/windows2019.yml

@@ -0,0 +1,27 @@
+run-name: Windows 2019 - ${{ (github.event.pull_request.title || 'scheduled/manual run') }}
+on:
+  workflow_dispatch:
+    inputs:
+      CUSTOM_REPOSITORY:
+        description: 'Custom repository (owner/repo)'
+        required: false
+      CUSTOM_REPOSITORY_COMMIT_HASH:
+        description: 'Commit hash'
+        required: false
+  pull_request_target:
+    types: labeled
+    paths:
+    - 'images/win/**'
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  Windows_2019:
+    if: ${{ (github.event.label.name == 'windows-all') || (github.event.label.name == 'windows-2019') || (inputs.CUSTOM_REPOSITORY != '' && inputs.CUSTOM_REPOSITORY_COMMIT_HASH != '') || (github.event_name == 'schedule') }}
+    uses: ./.github/workflows/ubuntu-win-generation.yml
+    with:
+      image_name: 'windows2019'
+      image_readme_name: 'Windows2019-Readme.md'
+      custom_repo: ${{ github.event.inputs.CUSTOM_REPOSITORY }}
+      custom_repo_commit_hash: ${{ github.event.inputs.CUSTOM_REPOSITORY_COMMIT_HASH }}
+    secrets: inherit

.github/workflows/windows2022.yml

@@ -0,0 +1,27 @@
+run-name: Windows 2022 - ${{ (github.event.pull_request.title || 'scheduled/manual run') }}
+on:
+  workflow_dispatch:
+    inputs:
+      CUSTOM_REPOSITORY:
+        description: 'Custom repository (owner/repo)'
+        required: false
+      CUSTOM_REPOSITORY_COMMIT_HASH:
+        description: 'Commit hash'
+        required: false
+  pull_request_target:
+    types: labeled
+    paths:
+    - 'images/win/**'
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  Windows_2022:
+    if: ${{ (github.event.label.name == 'windows-all') || (github.event.label.name == 'windows-2022') || (inputs.CUSTOM_REPOSITORY != '' && inputs.CUSTOM_REPOSITORY_COMMIT_HASH != '') || (github.event_name == 'schedule') }}
+    uses: ./.github/workflows/ubuntu-win-generation.yml
+    with:
+      image_name: 'windows2022'
+      image_readme_name: 'Windows2022-Readme.md'
+      custom_repo: ${{ github.event.inputs.CUSTOM_REPOSITORY }}
+      custom_repo_commit_hash: ${{ github.event.inputs.CUSTOM_REPOSITORY_COMMIT_HASH }}
+    secrets: inherit