From a8a908b63ee3ae03f1b7a8921c06873c1a830907 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Wed, 16 Sep 2020 19:44:58 +0300 Subject: [PATCH] Implement public CI for MacOS builds --- .../azure-pipelines/image-generation.yml | 8 +- .../azure-pipelines/ubuntu1604.yml | 0 .../azure-pipelines/ubuntu1804.yml | 0 .../azure-pipelines/ubuntu2004.yml | 0 .../azure-pipelines/windows2016.yml | 0 .../azure-pipelines/windows2019.yml | 0 images.CI/{ => linux-and-win}/build-image.ps1 | 0 images.CI/{ => linux-and-win}/cleanup.ps1 | 0 .../{ => linux-and-win}/create-release.ps1 | 0 .../azure-pipelines/image-generation.yml | 105 ++++++++++++++++++ images.CI/macos/azure-pipelines/macos1013.yml | 23 ++++ images.CI/macos/azure-pipelines/macos1014.yml | 23 ++++ images.CI/macos/azure-pipelines/macos1015.yml | 23 ++++ images.CI/macos/azure-pipelines/macos110.yml | 23 ++++ images.CI/macos/validate-contributor.ps1 | 63 +++++++++++ 15 files changed, 264 insertions(+), 4 deletions(-) rename images.CI/{ => linux-and-win}/azure-pipelines/image-generation.yml (93%) rename images.CI/{ => linux-and-win}/azure-pipelines/ubuntu1604.yml (100%) rename images.CI/{ => linux-and-win}/azure-pipelines/ubuntu1804.yml (100%) rename images.CI/{ => linux-and-win}/azure-pipelines/ubuntu2004.yml (100%) rename images.CI/{ => linux-and-win}/azure-pipelines/windows2016.yml (100%) rename images.CI/{ => linux-and-win}/azure-pipelines/windows2019.yml (100%) rename images.CI/{ => linux-and-win}/build-image.ps1 (100%) rename images.CI/{ => linux-and-win}/cleanup.ps1 (100%) rename images.CI/{ => linux-and-win}/create-release.ps1 (100%) create mode 100644 images.CI/macos/azure-pipelines/image-generation.yml create mode 100644 images.CI/macos/azure-pipelines/macos1013.yml create mode 100644 images.CI/macos/azure-pipelines/macos1014.yml create mode 100644 images.CI/macos/azure-pipelines/macos1015.yml create mode 100644 images.CI/macos/azure-pipelines/macos110.yml create mode 100644 images.CI/macos/validate-contributor.ps1 diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/linux-and-win/azure-pipelines/image-generation.yml similarity index 93% rename from images.CI/azure-pipelines/image-generation.yml rename to images.CI/linux-and-win/azure-pipelines/image-generation.yml index 70db48f1c..38aabd142 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/linux-and-win/azure-pipelines/image-generation.yml @@ -20,13 +20,13 @@ jobs: targetType: 'filePath' filePath: ./images.CI/download-repo.ps1 arguments: -RepoUrl $(CUSTOM_REPOSITORY_URL) ` - -RepoBranch $(CUSTOM_REPOSITORY_BRANCH) + -RepoBranch $(CUSTOM_REPOSITORY_BRANCH) - task: PowerShell@2 displayName: 'Build VM' inputs: targetType: filePath - filePath: ./images.CI/build-image.ps1 + filePath: ./images.CI/linux-and-win/build-image.ps1 arguments: -ResourcesNamePrefix $(Build.BuildId) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` @@ -45,7 +45,7 @@ jobs: displayName: 'Create release for VM deployment' inputs: targetType: filePath - filePath: ./images.CI/create-release.ps1 + filePath: ./images.CI/linux-and-win/create-release.ps1 arguments: -BuildId $(Build.BuildId) ` -Organization $(RELEASE_TARGET_ORGANIZATION) ` -DefinitionId $(RELEASE_TARGET_DEFINITION_ID) ` @@ -66,7 +66,7 @@ jobs: condition: always() inputs: targetType: filePath - filePath: ./images.CI/cleanup.ps1 + filePath: ./images.CI/linux-and-win/cleanup.ps1 arguments: -ResourcesNamePrefix $(Build.BuildId) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` diff --git a/images.CI/azure-pipelines/ubuntu1604.yml b/images.CI/linux-and-win/azure-pipelines/ubuntu1604.yml similarity index 100% rename from images.CI/azure-pipelines/ubuntu1604.yml rename to images.CI/linux-and-win/azure-pipelines/ubuntu1604.yml diff --git a/images.CI/azure-pipelines/ubuntu1804.yml b/images.CI/linux-and-win/azure-pipelines/ubuntu1804.yml similarity index 100% rename from images.CI/azure-pipelines/ubuntu1804.yml rename to images.CI/linux-and-win/azure-pipelines/ubuntu1804.yml diff --git a/images.CI/azure-pipelines/ubuntu2004.yml b/images.CI/linux-and-win/azure-pipelines/ubuntu2004.yml similarity index 100% rename from images.CI/azure-pipelines/ubuntu2004.yml rename to images.CI/linux-and-win/azure-pipelines/ubuntu2004.yml diff --git a/images.CI/azure-pipelines/windows2016.yml b/images.CI/linux-and-win/azure-pipelines/windows2016.yml similarity index 100% rename from images.CI/azure-pipelines/windows2016.yml rename to images.CI/linux-and-win/azure-pipelines/windows2016.yml diff --git a/images.CI/azure-pipelines/windows2019.yml b/images.CI/linux-and-win/azure-pipelines/windows2019.yml similarity index 100% rename from images.CI/azure-pipelines/windows2019.yml rename to images.CI/linux-and-win/azure-pipelines/windows2019.yml diff --git a/images.CI/build-image.ps1 b/images.CI/linux-and-win/build-image.ps1 similarity index 100% rename from images.CI/build-image.ps1 rename to images.CI/linux-and-win/build-image.ps1 diff --git a/images.CI/cleanup.ps1 b/images.CI/linux-and-win/cleanup.ps1 similarity index 100% rename from images.CI/cleanup.ps1 rename to images.CI/linux-and-win/cleanup.ps1 diff --git a/images.CI/create-release.ps1 b/images.CI/linux-and-win/create-release.ps1 similarity index 100% rename from images.CI/create-release.ps1 rename to images.CI/linux-and-win/create-release.ps1 diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml new file mode 100644 index 000000000..490cbda6d --- /dev/null +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -0,0 +1,105 @@ +jobs: +- job: Image_generation + displayName: Image Generation (${{ parameters.image_label }}) + timeoutInMinutes: 720 + pool: + name: Mac-Cloud Image Generation + variables: + - group: Mac-Cloud Image Generation + - group: Mac-Cloud Image Generation Key Vault + + steps: + - checkout: self + clean: true + fetchDepth: 1 + + - task: PowerShell@2 + displayName: 'Validate contributor permissions' + condition: startsWith(variables['Build.SourceBranch'], 'refs/pull/') + inputs: + targetType: 'filePath' + filePath: ./images.CI/macos/validate-contributor.ps1 + pwsh: true + arguments: -RepositoryName "$(Build.Repository.Name)" ` + -AccessToken "$(github-feed-token)" ` + -SourceBranch "$(Build.SourceBranch)" ` + -ContributorAllowList "$(CONTRIBUTOR_ALLOWLIST)" + + - task: PowerShell@2 + displayName: 'Download custom repository' + condition: and(ne(variables['CUSTOM_REPOSITORY_URL'], ''), ne(variables['CUSTOM_REPOSITORY_BRANCH'], '')) + inputs: + targetType: 'filePath' + filePath: ./images.CI/download-repo.ps1 + arguments: -RepoUrl $(CUSTOM_REPOSITORY_URL) ` + -RepoBranch $(CUSTOM_REPOSITORY_BRANCH) + + - task: DeleteFiles@1 + displayName: Clean up self-hosted machine + inputs: + SourceFolder: 'images/macos/provision/log/' + RemoveSourceFolder: true + + - pwsh: | + $SensitiveData = @( + 'IP address:', + 'Using ssh communicator to connect:' + ) + + packer build -on-error=abort ` + -var="vcenter_server=$(vcenter-server-v2)" ` + -var="vcenter_username=$(vcenter-username-v2)" ` + -var="vcenter_password=$(vcenter-password-v2)" ` + -var="vcenter_datacenter=$(vcenter-datacenter-v2)" ` + -var="cluster_or_esxi_host=$(esxi-cluster-v2)" ` + -var="esxi_datastore=${{ parameters.target_datastore }}" ` + -var="output_folder=$(output-folder)" ` + -var="vm_username=$(vm-username)" ` + -var="vm_password=$(vm-password)" ` + -var="build_id=$(Build.BuildNumber)" ` + -var="baseimage_name=${{ parameters.base_image_name }}" ` + -var="github_feed_token=$(github-feed-token)" ` + -var="xcode_install_user=$(xcode-installation-user)" ` + -var="xcode_install_password=$(xcode-installation-password)" ` + -color=false ` + ${{ parameters.template_path }} ` + | Where-Object { + #Filter sensitive data from Packer logs + $currentString = $_ + $sensitiveString = $SensitiveData | Where-Object { $currentString -match $_ } + $sensitiveString -eq $null + } + displayName: 'Build VM' + env: + PACKER_LOG: 0 + workingDirectory: 'images/macos' + + - bash: | + echo "Copy image output files" + cp -R "images/image-output/software-report/." "$(Build.ArtifactStagingDirectory)/" + + echo "Copy test results" + cp -R "images/image-output/tests/." "$(Common.TestResultsDirectory)/" + ls $(Common.TestResultsDirectory) + + echo "Put VM name to 'VM_Done_Name' file" + echo "$(Build.BuildNumber)" > "$(Build.ArtifactStagingDirectory)/VM_Done_Name" + displayName: Prepare artifact + + - bash: | + cat "$(Build.ArtifactStagingDirectory)/systeminfo.md" + displayName: Print software report + + - task: PublishBuildArtifacts@1 + inputs: + ArtifactName: 'Built_VM_Artifacts' + displayName: Publish Artifacts + + - task: PublishTestResults@2 + inputs: + testResultsFiles: '*.xml' + testResultsFormat: NUnit + searchFolder: '$(Common.TestResultsDirectory)' + failTaskOnFailedTests: true + displayName: Publish test results + condition: always() diff --git a/images.CI/macos/azure-pipelines/macos1013.yml b/images.CI/macos/azure-pipelines/macos1013.yml new file mode 100644 index 000000000..a7b9a4a19 --- /dev/null +++ b/images.CI/macos/azure-pipelines/macos1013.yml @@ -0,0 +1,23 @@ +name: macOS-10.13_$(date:yyyyMMdd)$(rev:.r)_unstable +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - main + always: true + +trigger: none +pr: + autoCancel: true + branches: + include: + - main + +jobs: +- template: image-generation.yml + parameters: + image_label: 'macOS High Sierra' + base_image_name: 'clean-macOS-10.13.6-380Gb-SIPoff_runner' + template_path: 'templates/macOS-10.13.json' + target_datastore: 'ds-image' diff --git a/images.CI/macos/azure-pipelines/macos1014.yml b/images.CI/macos/azure-pipelines/macos1014.yml new file mode 100644 index 000000000..21f7e48c1 --- /dev/null +++ b/images.CI/macos/azure-pipelines/macos1014.yml @@ -0,0 +1,23 @@ +name: macOS-10.14_$(date:yyyyMMdd)$(rev:.r)_unstable +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - main + always: true + +trigger: none +pr: + autoCancel: true + branches: + include: + - main + +jobs: +- template: image-generation.yml + parameters: + image_label: 'macOS Mojave' + base_image_name: 'clean-macOS-10.14-380Gb_runner' + template_path: 'templates/macOS-10.14.json' + target_datastore: 'ds-image' diff --git a/images.CI/macos/azure-pipelines/macos1015.yml b/images.CI/macos/azure-pipelines/macos1015.yml new file mode 100644 index 000000000..f42980cc4 --- /dev/null +++ b/images.CI/macos/azure-pipelines/macos1015.yml @@ -0,0 +1,23 @@ +name: macOS-10.15_$(date:yyyyMMdd)$(rev:.r)_unstable +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - main + always: true + +trigger: none +pr: + autoCancel: true + branches: + include: + - main + +jobs: +- template: image-generation.yml + parameters: + image_label: 'macOS Catalina' + base_image_name: 'clean-macOS-10.15-380Gb-runner' + template_path: 'templates/macOS-10.15.json' + target_datastore: 'ds-image' diff --git a/images.CI/macos/azure-pipelines/macos110.yml b/images.CI/macos/azure-pipelines/macos110.yml new file mode 100644 index 000000000..ada6dcfe1 --- /dev/null +++ b/images.CI/macos/azure-pipelines/macos110.yml @@ -0,0 +1,23 @@ +name: macOS-11.0_$(date:yyyyMMdd)$(rev:.r)_unstable +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - main + always: true + +trigger: none +pr: + autoCancel: true + branches: + include: + - main + +jobs: +- template: image-generation.yml + parameters: + image_label: 'macOS Big Sur' + base_image_name: 'clean-macOS-11.0-380Gb-runner' + template_path: 'templates/macOS-11.0.json' + target_datastore: 'ds-image' diff --git a/images.CI/macos/validate-contributor.ps1 b/images.CI/macos/validate-contributor.ps1 new file mode 100644 index 000000000..312a9ef7e --- /dev/null +++ b/images.CI/macos/validate-contributor.ps1 @@ -0,0 +1,63 @@ +param( + [Parameter(Mandatory)] [string] $RepositoryName, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [string] $SourceBranch, + [Parameter(Mandatory)] [string] $ContributorAllowList +) + +function Build-AuthHeader { + param( + [Parameter(Mandatory)] [string] $AccessToken + ) + + $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("'':${AccessToken}")) + return "Basic ${base64AuthInfo}" +} + +function Get-PullRequest { + param( + [Parameter(Mandatory)] [string] $RepositoryName, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [UInt32] $PullRequestNumber + ) + + $requestUrl = "https://api.github.com/repos/$RepositoryName/pulls/$PullRequestNumber" + $authHeader = Build-AuthHeader -AccessToken $AccessToken + + $params = @{ + Method = "GET" + ContentType = "application/json" + Uri = $requestUrl + Headers = @{ Authorization = $authHeader } + } + + return Invoke-RestMethod @params +} + +function Validate-ContributorPermissions { + param( + [Parameter(Mandatory)] [string] $ContributorAllowList, + [Parameter(Mandatory)] [string] $ContributorName + ) + + $allowedContributors = $ContributorAllowList.Split(",").Trim() + $validСontributor = $allowedContributors | Where-Object { $_ -eq $ContributorName } ` + | Select-Object -First 1 + + if (-not $validСontributor) { + Write-Host "Failed to start this build. '$ContributorName' is an unknown contributor" + Write-Host "Please add '$ContributorName' to the allowed list to run builds" + exit 1 + } +} + +$pullRequestNumber = $SourceBranch.Split("/")[2] + +$pullRequestInfo = Get-PullRequest -RepositoryName $RepositoryName ` + -AccessToken $AccessToken ` + -PullRequestNumber $pullRequestNumber + +$contributorName = $pullRequestInfo.user.login + +Validate-ContributorPermissions -ContributorAllowList $ContributorAllowList ` + -ContributorName $contributorName \ No newline at end of file