diff --git a/.github/workflows/macos-generation.yml b/.github/workflows/macos-generation.yml new file mode 100644 index 000000000..e91f4470c --- /dev/null +++ b/.github/workflows/macos-generation.yml @@ -0,0 +1,212 @@ +name: macOS image generation +on: + workflow_call: + inputs: + image_label: + type: string + description: macOS codename + required: true + base_image_name: + type: string + description: Base clean image + required: true + template_path: + type: string + description: Packer template path + required: true + target_datastore: + type: string + description: Image datastore + required: true + custom_repo: + type: string + description: Custom repo to checkout + required: false + custom_repo_commit_hash: + type: string + description: Custom repo commit hash + required: false + +env: + KEYVAULT: imagegeneration + ESXI_CLUSTER: mcv2-build-unstable + VCENTER_DATACENTER: imagegen + OUTPUT_FOLDER: mms-output + BUILD_DATASTORE: ds-image + +defaults: + run: + shell: pwsh + +jobs: + build: + runs-on: macos-vmware + timeout-minutes: 1200 + steps: + - uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Set image variables + run: | + $currentDate = Get-Date -Format "yyyyMMdd" + $templatePath = "${{ inputs.template_path }}" + $osName = $(($templatePath.Split("/")[-1]).Split(".")[0]) + $virtualMachineName = "${osName}_${currentDate}_unstable.${{ github.run_id }}.${{ github.run_attempt }}" + $GitHubFeed = az keyvault secret show -n "github-feed-token" --vault-name "${{ env.KEYVAULT }}" --query value -o tsv + $VIUserName = az keyvault secret show -n "vcenter-username-v2" --vault-name "${{ env.KEYVAULT }}" --query value -o tsv + $VIPassword = az keyvault secret show -n "vcenter-password-v2" --vault-name "${{ env.KEYVAULT }}" --query value -o tsv + $XcodeUser = az keyvault secret show -n "xcode-installation-user" --vault-name "${{ env.KEYVAULT }}" --query value -o tsv + $XcodePassword = az keyvault secret show -n "xcode-installation-password" --vault-name "${{ env.KEYVAULT }}" --query value -o tsv + echo "::add-mask::$GitHubFeed" + echo "::add-mask::$VIUserName" + echo "::add-mask::$VIPassword" + echo "::add-mask::$XcodeUser" + echo "::add-mask::$XcodePassword" + "GH_FEED=$GitHubFeed" | Out-File -Append -FilePath $env:GITHUB_ENV + "VI_USER_NAME=$VIUserName" | Out-File -Append -FilePath $env:GITHUB_ENV + "VI_PASSWORD=$VIPassword" | Out-File -Append -FilePath $env:GITHUB_ENV + "XCODE_USER=$XcodeUser" | Out-File -Append -FilePath $env:GITHUB_ENV + "XCODE_PASSWORD=$XcodePassword" | Out-File -Append -FilePath $env:GITHUB_ENV + "VM_NAME=$virtualMachineName" | Out-File -Append -FilePath $env:GITHUB_ENV + + - name: Determine checkout type + run: | + if ("${{ inputs.custom_repo }}" -and "${{ inputs.custom_repo_commit_hash }}") { + $checkoutType = "custom_repo" + } elseif (("${{ github.event_name }}" -eq "pull_request_target") -and ("${{ github.event.action }}" -eq "labeled" )) { + $checkoutType = "pull_request" + } else { + $checkoutType = "main" + } + "CHECKOUT_TYPE=$checkoutType" | Out-File -Append $env:GITHUB_ENV + + - name: Checkout repository + if: ${{ env.CHECKOUT_TYPE == 'main' }} + uses: actions/checkout@v3 + with: + repository: actions/runner-images + + - name: Checkout PR + if: ${{ env.CHECKOUT_TYPE == 'pull_request' }} + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Checkout custom repository + if: ${{ env.CHECKOUT_TYPE == 'custom_repo' }} + uses: actions/checkout@v3 + with: + repository: '${{ inputs.custom_repo }}' + ref: '${{ inputs.custom_repo_commit_hash }}' + + - name: Validate contributor permissions + if: ${{ github.event_name == 'pull_request_target' }} + run: | + + [string]$contributorAllowList = "${{ vars.CONTRIBUTOR_ALLOWLIST }}" + ./images.CI/macos/validate-contributor.ps1 ` + -RepositoryName ${{ github.repository }} ` + -AccessToken ${{ env.GH_FEED }} ` + -SourceBranch "refs/pull/${{ github.event.pull_request.number }}/merge" ` + -ContributorAllowList $contributorAllowList + + - name: Select datastore + run: | + ./images.CI/macos/select-datastore.ps1 ` + -VMName "${{ env.VM_NAME }}" ` + -VIServer ${{ secrets.VISERVER_V2 }} ` + -VIUserName ${{ env.VI_USER_NAME }} ` + -VIPassword ${{ env.VI_PASSWORD }} ` + -Cluster ${{ env.ESXI_CLUSTER }} + + - name: Build VM + run: | + $SensitiveData = @( + 'IP address:', + 'Using ssh communicator to connect:' + ) + packer build -on-error=abort ` + -var="vcenter_server=${{ secrets.VISERVER_V2 }}" ` + -var="vcenter_username=${{ env.VI_USER_NAME }}" ` + -var="vcenter_password=${{ env.VI_PASSWORD }}" ` + -var="vcenter_datacenter=${{ env.VCENTER_DATACENTER }}" ` + -var="cluster_or_esxi_host=${{ env.ESXI_CLUSTER }}" ` + -var="esxi_datastore=${{ env.BUILD_DATASTORE }}" ` + -var="output_folder=${{ env.OUTPUT_FOLDER }}" ` + -var="vm_username=${{ secrets.VM_USERNAME }}" ` + -var="vm_password=${{ secrets.VM_PASSWORD }}" ` + -var="github_api_pat=${{ secrets.GH_FEED_TOKEN }}" ` + -var="build_id=${{ env.VM_NAME }}" ` + -var="baseimage_name=${{ inputs.base_image_name }}" ` + -var="xcode_install_user=${{ env.XCODE_USER }}" ` + -var="xcode_install_password=${{ env.XCODE_PASSWORD }}" ` + -color=false ` + ${{ inputs.template_path }} ` + | Where-Object { + #Filter sensitive data from Packer logs + $currentString = $_ + $sensitiveString = $SensitiveData | Where-Object { $currentString -match $_ } + $sensitiveString -eq $null + } + working-directory: images/macos + env: + PACKER_LOG: 1 + PACKER_LOG_PATH: ${{ runner.temp }}/packer-log.txt + + - name: Prepare artifact + shell: bash + run: | + echo "Preparing artifact directory" + mkdir -p ${{ runner.temp }}/artifacts + + echo "Copy image output files" + cp -R "images/image-output/software-report/." "${{ runner.temp }}/artifacts" + + echo "Put VM name to 'VM_Done_Name' file" + echo "${{ env.VM_NAME }}" > "${{ runner.temp }}/artifacts/VM_Done_Name" + + - name: Print markdown software report + run: | + Get-Content "${{ runner.temp }}/artifacts/systeminfo.md" + + - name: Print json software report + run: | + Get-Content "${{ runner.temp }}/artifacts/systeminfo.json" + + - name: Publish Artifacts + uses: actions/upload-artifact@v3 + with: + name: Built_VM_Artifacts + path: ${{ runner.temp }}/artifacts/ + + - name: Print provisioners duration + run: | + ./images.CI/measure-provisioners-duration.ps1 ` + -PackerLogPath "${{ runner.temp }}/packer-log.txt" ` + -PrintTopNLongest 25 + + - name: Move vm to cold storage and clear datastore tag + run: | + $cpuCount = 3 + $coresPerSocketCount = 3 + $memory = 14336 + + ./images.CI/macos/move-vm.ps1 ` + -VMName "${{ env.VM_NAME }}" ` + -TargetDataStore "${{ inputs.target_datastore }}" ` + -VIServer "${{ secrets.VISERVER_V2 }}" ` + -VIUserName "${{ env.VI_USER_NAME }}" ` + -VIPassword "${{ env.VI_PASSWORD }}" ` + -CpuCount "$cpuCount" ` + -CoresPerSocketCount "$coresPerSocketCount" ` + -Memory "$memory" + + - name: Destroy VM (if build canceled only) + if: ${{ cancelled() }} + run: | + ./images.CI/macos/destroy-vm.ps1 ` + -VMName "${{ env.VM_NAME }}" ` + -VIServer "${{ secrets.VISERVER_V2 }}" ` + -VIUserName "${{ env.VI_USER_NAME }}" ` + -VIPassword "${{ env.VI_PASSWORD }}" \ No newline at end of file diff --git a/.github/workflows/macos11.yml b/.github/workflows/macos11.yml new file mode 100644 index 000000000..daf1f3e70 --- /dev/null +++ b/.github/workflows/macos11.yml @@ -0,0 +1,30 @@ +run-name: macOS-11_unstable.${{ github.run_id }}.${{ github.run_attempt }} +on: + workflow_dispatch: + inputs: + CUSTOM_REPOSITORY: + description: 'Custom repository (owner/repo)' + required: false + CUSTOM_REPOSITORY_COMMIT_HASH: + description: 'Commit hash' + required: false + pull_request_target: + types: labeled + paths: + - 'images/macos/**' + schedule: + - cron: '0 0 * * *' + +jobs: + macOS_11: + if: ${{ (github.event.label.name == 'macos-all') || (github.event.label.name == 'macos-11') || (inputs.CUSTOM_REPOSITORY != '' && inputs.CUSTOM_REPOSITORY_COMMIT_HASH != '') || (github.event_name == 'schedule') }} + name: macOS-11_unstable.${{ github.run_id }}.${{ github.run_attempt }} + uses: ./.github/workflows/macos-generation.yml + with: + image_label: 'macOS Big Sur' + base_image_name: 'clean-macOS-11-380Gb-runner' + template_path: 'templates/macOS-11.json' + target_datastore: 'ds-image' + custom_repo: ${{ github.event.inputs.CUSTOM_REPOSITORY }} + custom_repo_hash: ${{ github.event.inputs.CUSTOM_REPOSITORY_COMMIT_HASH }} + secrets: inherit diff --git a/.github/workflows/macos12.yml b/.github/workflows/macos12.yml new file mode 100644 index 000000000..0ece0eb87 --- /dev/null +++ b/.github/workflows/macos12.yml @@ -0,0 +1,30 @@ +run-name: macOS-12_unstable.${{ github.run_id }}.${{ github.run_attempt }} +on: + workflow_dispatch: + inputs: + CUSTOM_REPOSITORY: + description: 'Custom repository (owner/repo)' + required: false + CUSTOM_REPOSITORY_COMMIT_HASH: + description: 'Commit hash' + required: false + pull_request_target: + types: labeled + paths: + - 'images/macos/**' + schedule: + - cron: '0 0 * * *' + +jobs: + macOS_12: + if: ${{ (github.event.label.name == 'macos-all') || (github.event.label.name == 'macos-12') || (inputs.CUSTOM_REPOSITORY != '' && inputs.CUSTOM_REPOSITORY_COMMIT_HASH != '') || (github.event_name == 'schedule') }} + name: macOS-12_unstable.${{ github.run_id }}.${{ github.run_attempt }} + uses: ./.github/workflows/macos-generation.yml + with: + image_label: 'macOS Monterey' + base_image_name: 'clean-macOS-12-380Gb-runner' + template_path: 'templates/macOS-12.json' + target_datastore: 'ds-image' + custom_repo: ${{ github.event.inputs.CUSTOM_REPOSITORY }} + custom_repo_commit_hash: ${{ github.event.inputs.CUSTOM_REPOSITORY_COMMIT_HASH }} + secrets: inherit diff --git a/.github/workflows/ubuntu-win-generation.yml b/.github/workflows/ubuntu-win-generation.yml new file mode 100644 index 000000000..9acf66438 --- /dev/null +++ b/.github/workflows/ubuntu-win-generation.yml @@ -0,0 +1,128 @@ +name: MMS image generation +on: + workflow_call: + inputs: + image_name: + type: string + description: An OS image to build + required: true + image_readme_name: + type: string + description: README file path + required: true + custom_repo: + type: string + description: Custom repo to checkout + required: false + custom_repo_commit_hash: + type: string + description: Custom repo commit hash + required: false +defaults: + run: + shell: pwsh + +jobs: + build: + runs-on: azure-builds + timeout-minutes: 1200 + steps: + - name: Determine checkout type + run: | + if ("${{ inputs.custom_repo }}" -and "${{ inputs.custom_repo_commit_hash }}") { + $checkoutType = "custom_repo" + } elseif (("${{ github.event_name }}" -eq "pull_request_target") -and ("${{ github.event.action }}" -eq "labeled" )) { + $checkoutType = "pull_request" + } else { + $checkoutType = "main" + } + "CHECKOUT_TYPE=$checkoutType" | Out-File -Append $env:GITHUB_ENV + + - name: Checkout repository + if: ${{ env.CHECKOUT_TYPE == 'main' }} + uses: actions/checkout@v3 + with: + repository: actions/runner-images + + - name: Checkout PR + if: ${{ env.CHECKOUT_TYPE == 'pull_request' }} + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Checkout custom repository + if: ${{ env.CHECKOUT_TYPE == 'custom_repo' }} + uses: actions/checkout@v3 + with: + repository: '${{ inputs.custom_repo }}' + ref: '${{ inputs.custom_repo_commit_hash }}' + + - name: Set image varibles + run: | + $ImageType = "${{ inputs.image_name }}" + + if ($ImageType.StartsWith("ubuntu")) { $TemplateDirectoryName = "linux" } else { $TemplateDirectoryName = "win" } + + $TemplateDirectoryPath = Join-Path "images" $TemplateDirectoryName | Resolve-Path + $TemplatePath = Join-Path $TemplateDirectoryPath "$ImageType.pkr.hcl" + + if ( -not (Test-Path $TemplatePath) ) { + $TemplatePath = Join-Path $TemplateDirectoryPath "$ImageType.json" + } + + "TemplatePath=$TemplatePath" | Out-File -Append -FilePath $env:GITHUB_ENV + "TemplateDirectoryPath=$TemplateDirectoryPath" | Out-File -Append -FilePath $env:GITHUB_ENV + "ImageType=$ImageType" | Out-File -Append -FilePath $env:GITHUB_ENV + + - name: Build image + run: | + ./images.CI/linux-and-win/build-image.ps1 ` + -TemplatePath ${{ env.TemplatePath }} ` + -ClientId ${{ secrets.CLIENT_ID }} ` + -ClientSecret ${{ secrets.CLIENT_SECRET }} ` + -Location ${{ secrets.AZURE_LOCATION }} ` + -ResourcesNamePrefix ${{ github.run_number }} ` + -ResourceGroup ${{ secrets.AZURE_RESOURCE_GROUP }} ` + -StorageAccount ${{ secrets.AZURE_STORAGE_ACCOUNT }} ` + -SubscriptionId ${{ secrets.AZURE_SUBSCRIPTION }} ` + -TenantId ${{ secrets.AZURE_TENANT }} ` + -VirtualNetworkName ${{ secrets.BUILD_AGENT_VNET_NAME }} ` + -VirtualNetworkSubnet ${{ secrets.BUILD_AGENT_SUBNET_NAME }} ` + -VirtualNetworkRG ${{ secrets.BUILD_AGENT_VNET_RESOURCE_GROUP }} ` + env: + PACKER_LOG: 1 + PACKER_LOG_PATH: ${{ runner.temp }}/packer-log.txt + RUN_VALIDATION_FLAG: true + + - name: Output Readme file content + run: | + Get-Content -Path (Join-Path "$env:TemplateDirectoryPath" "${{ inputs.image_readme_name }}") + + - name: Print provisioners duration + run: | + ./images.CI/measure-provisioners-duration.ps1 ` + -PackerLogPath "${{ runner.temp }}/packer-log.txt" ` + -PrefixToPathTrim ${{ env.TemplateDirectoryPath }} ` + -PrintTopNLongest 25 + + - name: Create release for VM deployment + run: | + ./images.CI/linux-and-win/create-release.ps1 ` + -BuildId ${{ github.run_number }} ` + -Organization ${{ secrets.RELEASE_TARGET_ORGANIZATION }} ` + -DefinitionId ${{ secrets.RELEASE_TARGET_DEFINITION_ID }} ` + -Project ${{ secrets.RELEASE_TARGET_PROJECT }} ` + -ImageName ${{ env.ImageType }} ` + -AccessToken ${{ secrets.RELEASE_TARGET_TOKEN }} + + - name: Clean up resources + if: ${{ always() }} + run: | + ./images.CI/linux-and-win/cleanup.ps1 ` + -ResourcesNamePrefix ${{ github.run_number }} ` + -Image ${{ env.ImageType }} ` + -StorageAccount ${{ secrets.AZURE_STORAGE_ACCOUNT }} ` + -SubscriptionId ${{ secrets.AZURE_SUBSCRIPTION }} ` + -ClientId ${{ secrets.CLIENT_ID }} ` + -ClientSecret ${{ secrets.CLIENT_SECRET }} ` + -TenantId ${{ secrets.AZURE_TENANT }} diff --git a/.github/workflows/ubuntu2004.yml b/.github/workflows/ubuntu2004.yml new file mode 100644 index 000000000..8476ab7d3 --- /dev/null +++ b/.github/workflows/ubuntu2004.yml @@ -0,0 +1,27 @@ +run-name: Ubuntu20.04 - ${{ (github.event.pull_request.title || 'scheduled/manual run') }} +on: + workflow_dispatch: + inputs: + CUSTOM_REPOSITORY: + description: 'Custom repository (owner/repo)' + required: false + CUSTOM_REPOSITORY_COMMIT_HASH: + description: 'Commit hash' + required: false + pull_request_target: + types: labeled + paths: + - 'images/linux/**' + schedule: + - cron: '0 0 * * *' + +jobs: + Ubuntu_2004: + if: ${{ (github.event.label.name == 'ubuntu-all') || (github.event.label.name == 'ubuntu-2004') || (inputs.CUSTOM_REPOSITORY != '' && inputs.CUSTOM_REPOSITORY_COMMIT_HASH != '') || (github.event_name == 'schedule') }} + uses: ./.github/workflows/ubuntu-win-generation.yml + with: + image_name: 'ubuntu2004' + image_readme_name: 'Ubuntu2004-Readme.md' + custom_repo: ${{ github.event.inputs.CUSTOM_REPOSITORY }} + custom_repo_commit_hash: ${{ github.event.inputs.CUSTOM_REPOSITORY_COMMIT_HASH }} + secrets: inherit diff --git a/.github/workflows/ubuntu2204.yml b/.github/workflows/ubuntu2204.yml new file mode 100644 index 000000000..446427ada --- /dev/null +++ b/.github/workflows/ubuntu2204.yml @@ -0,0 +1,27 @@ +run-name: Ubuntu22.04 - ${{ (github.event.pull_request.title || 'scheduled/manual run') }} +on: + workflow_dispatch: + inputs: + CUSTOM_REPOSITORY: + description: 'Custom repository (owner/repo)' + required: false + CUSTOM_REPOSITORY_COMMIT_HASH: + description: 'Commit hash' + required: false + pull_request_target: + types: labeled + paths: + - 'images/linux/**' + schedule: + - cron: '0 0 * * *' + +jobs: + Ubuntu_2204: + if: ${{ (github.event.label.name == 'ubuntu-all') || (github.event.label.name == 'ubuntu-2204') || (inputs.CUSTOM_REPOSITORY != '' && inputs.CUSTOM_REPOSITORY_COMMIT_HASH != '') || (github.event_name == 'schedule') }} + uses: ./.github/workflows/ubuntu-win-generation.yml + with: + image_name: 'ubuntu2204' + image_readme_name: 'Ubuntu2204-Readme.md' + custom_repo: ${{ github.event.inputs.CUSTOM_REPOSITORY }} + custom_repo_commit_hash: ${{ github.event.inputs.CUSTOM_REPOSITORY_COMMIT_HASH }} + secrets: inherit diff --git a/.github/workflows/windows2019.yml b/.github/workflows/windows2019.yml new file mode 100644 index 000000000..2de6404c6 --- /dev/null +++ b/.github/workflows/windows2019.yml @@ -0,0 +1,27 @@ +run-name: Windows 2019 - ${{ (github.event.pull_request.title || 'scheduled/manual run') }} +on: + workflow_dispatch: + inputs: + CUSTOM_REPOSITORY: + description: 'Custom repository (owner/repo)' + required: false + CUSTOM_REPOSITORY_COMMIT_HASH: + description: 'Commit hash' + required: false + pull_request_target: + types: labeled + paths: + - 'images/win/**' + schedule: + - cron: '0 0 * * *' + +jobs: + Windows_2019: + if: ${{ (github.event.label.name == 'windows-all') || (github.event.label.name == 'windows-2019') || (inputs.CUSTOM_REPOSITORY != '' && inputs.CUSTOM_REPOSITORY_COMMIT_HASH != '') || (github.event_name == 'schedule') }} + uses: ./.github/workflows/ubuntu-win-generation.yml + with: + image_name: 'windows2019' + image_readme_name: 'Windows2019-Readme.md' + custom_repo: ${{ github.event.inputs.CUSTOM_REPOSITORY }} + custom_repo_commit_hash: ${{ github.event.inputs.CUSTOM_REPOSITORY_COMMIT_HASH }} + secrets: inherit diff --git a/.github/workflows/windows2022.yml b/.github/workflows/windows2022.yml new file mode 100644 index 000000000..ef5e5d3d5 --- /dev/null +++ b/.github/workflows/windows2022.yml @@ -0,0 +1,27 @@ +run-name: Windows 2022 - ${{ (github.event.pull_request.title || 'scheduled/manual run') }} +on: + workflow_dispatch: + inputs: + CUSTOM_REPOSITORY: + description: 'Custom repository (owner/repo)' + required: false + CUSTOM_REPOSITORY_COMMIT_HASH: + description: 'Commit hash' + required: false + pull_request_target: + types: labeled + paths: + - 'images/win/**' + schedule: + - cron: '0 0 * * *' + +jobs: + Windows_2022: + if: ${{ (github.event.label.name == 'windows-all') || (github.event.label.name == 'windows-2022') || (inputs.CUSTOM_REPOSITORY != '' && inputs.CUSTOM_REPOSITORY_COMMIT_HASH != '') || (github.event_name == 'schedule') }} + uses: ./.github/workflows/ubuntu-win-generation.yml + with: + image_name: 'windows2022' + image_readme_name: 'Windows2022-Readme.md' + custom_repo: ${{ github.event.inputs.CUSTOM_REPOSITORY }} + custom_repo_commit_hash: ${{ github.event.inputs.CUSTOM_REPOSITORY_COMMIT_HASH }} + secrets: inherit