From 962aa94f3ad04745ba011c20abcb5a4adac09014 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Fri, 11 Sep 2020 17:34:56 +0300 Subject: [PATCH 01/17] Add CI for macos --- .../azure-pipelines/image-generation.yml | 2 +- .../azure-pipelines/ubuntu1604.yml | 0 .../azure-pipelines/ubuntu1804.yml | 0 .../azure-pipelines/ubuntu2004.yml | 0 .../azure-pipelines/windows2016.yml | 0 .../azure-pipelines/windows2019.yml | 0 images.CI/{ => linux-and-win}/build-image.ps1 | 0 images.CI/{ => linux-and-win}/cleanup.ps1 | 0 .../{ => linux-and-win}/create-release.ps1 | 0 .../credscan-exclusions.json | 0 .../{ => linux-and-win}/download-repo.ps1 | 0 .../azure-pipelines/image-generation.yml | 84 +++++++++++++++++++ images.CI/macos/azure-pipelines/macos1013.yml | 23 +++++ images.CI/macos/azure-pipelines/macos1014.yml | 23 +++++ images.CI/macos/azure-pipelines/macos1015.yml | 23 +++++ 15 files changed, 154 insertions(+), 1 deletion(-) rename images.CI/{ => linux-and-win}/azure-pipelines/image-generation.yml (98%) rename images.CI/{ => linux-and-win}/azure-pipelines/ubuntu1604.yml (100%) rename images.CI/{ => linux-and-win}/azure-pipelines/ubuntu1804.yml (100%) rename images.CI/{ => linux-and-win}/azure-pipelines/ubuntu2004.yml (100%) rename images.CI/{ => linux-and-win}/azure-pipelines/windows2016.yml (100%) rename images.CI/{ => linux-and-win}/azure-pipelines/windows2019.yml (100%) rename images.CI/{ => linux-and-win}/build-image.ps1 (100%) rename images.CI/{ => linux-and-win}/cleanup.ps1 (100%) rename images.CI/{ => linux-and-win}/create-release.ps1 (100%) rename images.CI/{ => linux-and-win}/credscan-exclusions.json (100%) rename images.CI/{ => linux-and-win}/download-repo.ps1 (100%) create mode 100644 images.CI/macos/azure-pipelines/image-generation.yml create mode 100644 images.CI/macos/azure-pipelines/macos1013.yml create mode 100644 images.CI/macos/azure-pipelines/macos1014.yml create mode 100644 images.CI/macos/azure-pipelines/macos1015.yml diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/linux-and-win/azure-pipelines/image-generation.yml similarity index 98% rename from images.CI/azure-pipelines/image-generation.yml rename to images.CI/linux-and-win/azure-pipelines/image-generation.yml index 70db48f1c..30499ce3f 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/linux-and-win/azure-pipelines/image-generation.yml @@ -20,7 +20,7 @@ jobs: targetType: 'filePath' filePath: ./images.CI/download-repo.ps1 arguments: -RepoUrl $(CUSTOM_REPOSITORY_URL) ` - -RepoBranch $(CUSTOM_REPOSITORY_BRANCH) + -RepoBranch $(CUSTOM_REPOSITORY_BRANCH) - task: PowerShell@2 displayName: 'Build VM' diff --git a/images.CI/azure-pipelines/ubuntu1604.yml b/images.CI/linux-and-win/azure-pipelines/ubuntu1604.yml similarity index 100% rename from images.CI/azure-pipelines/ubuntu1604.yml rename to images.CI/linux-and-win/azure-pipelines/ubuntu1604.yml diff --git a/images.CI/azure-pipelines/ubuntu1804.yml b/images.CI/linux-and-win/azure-pipelines/ubuntu1804.yml similarity index 100% rename from images.CI/azure-pipelines/ubuntu1804.yml rename to images.CI/linux-and-win/azure-pipelines/ubuntu1804.yml diff --git a/images.CI/azure-pipelines/ubuntu2004.yml b/images.CI/linux-and-win/azure-pipelines/ubuntu2004.yml similarity index 100% rename from images.CI/azure-pipelines/ubuntu2004.yml rename to images.CI/linux-and-win/azure-pipelines/ubuntu2004.yml diff --git a/images.CI/azure-pipelines/windows2016.yml b/images.CI/linux-and-win/azure-pipelines/windows2016.yml similarity index 100% rename from images.CI/azure-pipelines/windows2016.yml rename to images.CI/linux-and-win/azure-pipelines/windows2016.yml diff --git a/images.CI/azure-pipelines/windows2019.yml b/images.CI/linux-and-win/azure-pipelines/windows2019.yml similarity index 100% rename from images.CI/azure-pipelines/windows2019.yml rename to images.CI/linux-and-win/azure-pipelines/windows2019.yml diff --git a/images.CI/build-image.ps1 b/images.CI/linux-and-win/build-image.ps1 similarity index 100% rename from images.CI/build-image.ps1 rename to images.CI/linux-and-win/build-image.ps1 diff --git a/images.CI/cleanup.ps1 b/images.CI/linux-and-win/cleanup.ps1 similarity index 100% rename from images.CI/cleanup.ps1 rename to images.CI/linux-and-win/cleanup.ps1 diff --git a/images.CI/create-release.ps1 b/images.CI/linux-and-win/create-release.ps1 similarity index 100% rename from images.CI/create-release.ps1 rename to images.CI/linux-and-win/create-release.ps1 diff --git a/images.CI/credscan-exclusions.json b/images.CI/linux-and-win/credscan-exclusions.json similarity index 100% rename from images.CI/credscan-exclusions.json rename to images.CI/linux-and-win/credscan-exclusions.json diff --git a/images.CI/download-repo.ps1 b/images.CI/linux-and-win/download-repo.ps1 similarity index 100% rename from images.CI/download-repo.ps1 rename to images.CI/linux-and-win/download-repo.ps1 diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml new file mode 100644 index 000000000..b4af41203 --- /dev/null +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -0,0 +1,84 @@ +jobs: +- job: Image_generation + displayName: Image Generation (${{ parameters.image_label }}) + timeoutInMinutes: 720 + pool: + name: Mac-Cloud V2 Image Generation + variables: + - group: Mac-Cloud Image Generation + - group: Mac-Cloud Image Generation Key Vault + + steps: + - checkout: self + clean: true + fetchDepth: 1 + + - task: PowerShell@2 + displayName: 'Download custom repository' + condition: and(ne(variables['CUSTOM_REPOSITORY_URL'], ''), ne(variables['CUSTOM_REPOSITORY_BRANCH'], '')) + inputs: + targetType: 'filePath' + filePath: ./images.CI/download-repo.ps1 + arguments: -RepoUrl $(CUSTOM_REPOSITORY_URL) ` + -RepoBranch $(CUSTOM_REPOSITORY_BRANCH) + + - task: DeleteFiles@1 + displayName: Clean up self-hosted machine + inputs: + SourceFolder: 'image-generation/packer/provision/log/' + RemoveSourceFolder: true + + - bash: | + export PACKER_LOG=0 + packer build -on-error=abort \ + -var="vcenter_server=$(vcenter_server_v2)" \ + -var="vcenter_username=$(vcenter_username_v2)" \ + -var="vcenter_password=$(vcenter_password_v2)" \ + -var="vcenter_datacenter=$(vcenter_datacenter_v2)" \ + -var="cluster_or_esxi_host=$(esxi_cluster_v2)" \ + -var="esxi_datastore=${{ parameters.target_datastore }}" \ + -var="output_folder=mms-output" \ + -var="vm_username=$(vm_username)" \ + -var="vm_password=$(vm_password)" \ + -var="build_id=$(Build.BuildNumber)" \ + -var="baseimage_name=${{ parameters.base_image_name }}" \ + -var="azure_storage_sas=$(AZURE_STORAGE_SAS)" \ + -var="azure_storage_account=$(AZURE_STORAGE_ACCOUNT)" \ + -var="build_assets_url=$(appcenter_build_assets_url)" \ + -var="github_feed_token=$(GITHUB_FEED_TOKEN)" \ + -var="xcode_install_user=$(xcode-installation-user)" \ + -var="xcode_install_password=$(xcode-installation-password)" \ + -color=false \ + ${{ parameters.template_path }} + workingDirectory: 'image-generation/packer' + displayName: Build image + + - bash: | + echo "Copy image output files" + cp -R "image-generation/image-output/software-report/." "$(Build.ArtifactStagingDirectory)/" + + echo "Copy test results" + cp -R "image-generation/image-output/tests/." "$(Common.TestResultsDirectory)/" + ls $(Common.TestResultsDirectory) + + echo "Put VM name to 'VM_Done_Name' file" + echo "$(Build.BuildNumber)" > "$(Build.ArtifactStagingDirectory)/VM_Done_Name" + displayName: Prepare artifact + + - bash: | + cat "$(Build.ArtifactStagingDirectory)/systeminfo.md" + displayName: Print software report + + - task: PublishBuildArtifacts@1 + inputs: + ArtifactName: 'Built_VM_Artifacts' + displayName: Publish Artifacts + + - task: PublishTestResults@2 + inputs: + testResultsFiles: '*.xml' + testResultsFormat: NUnit + searchFolder: '$(Common.TestResultsDirectory)' + failTaskOnFailedTests: true + displayName: Publish test results + condition: always() diff --git a/images.CI/macos/azure-pipelines/macos1013.yml b/images.CI/macos/azure-pipelines/macos1013.yml new file mode 100644 index 000000000..04a76346d --- /dev/null +++ b/images.CI/macos/azure-pipelines/macos1013.yml @@ -0,0 +1,23 @@ +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - main + always: true + +trigger: none +pr: + autoCancel: true + branches: + include: + - main + +jobs: +- template: image-generation.yml + parameters: + image_label: 'macOS High Sierra' + base_image_name: 'clean-macOS-10.13.6-380Gb-SIPoff_runner' + template_path: 'templates/macOS-10.13.json' + target_datastore: 'ds-image' + stable: false diff --git a/images.CI/macos/azure-pipelines/macos1014.yml b/images.CI/macos/azure-pipelines/macos1014.yml new file mode 100644 index 000000000..8149c55c2 --- /dev/null +++ b/images.CI/macos/azure-pipelines/macos1014.yml @@ -0,0 +1,23 @@ +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - main + always: true + +trigger: none +pr: + autoCancel: true + branches: + include: + - main + +jobs: +- template: image-generation.yml + parameters: + image_label: 'macOS Mojave' + base_image_name: 'clean-macOS-10.14-380Gb_runner' + template_path: 'templates/macOS-10.14.json' + target_datastore: 'ds-image' + stable: false diff --git a/images.CI/macos/azure-pipelines/macos1015.yml b/images.CI/macos/azure-pipelines/macos1015.yml new file mode 100644 index 000000000..7fc37016a --- /dev/null +++ b/images.CI/macos/azure-pipelines/macos1015.yml @@ -0,0 +1,23 @@ +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - main + always: true + +trigger: none +pr: + autoCancel: true + branches: + include: + - main + +jobs: +- template: image-generation.yml + parameters: + image_label: 'macOS Catalina' + base_image_name: 'clean-macOS-10.15-380Gb-runner' + template_path: 'templates/macOS-10.15.json' + target_datastore: 'ds-image' + stable: false From 77330e4553eff9fc7971341ed8a2114525431a22 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Fri, 11 Sep 2020 18:02:49 +0300 Subject: [PATCH 02/17] fix path --- images.CI/macos/azure-pipelines/image-generation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index b4af41203..40cf4d43a 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -25,7 +25,7 @@ jobs: - task: DeleteFiles@1 displayName: Clean up self-hosted machine inputs: - SourceFolder: 'image-generation/packer/provision/log/' + SourceFolder: 'images/macos/provision/log/' RemoveSourceFolder: true - bash: | @@ -50,7 +50,7 @@ jobs: -var="xcode_install_password=$(xcode-installation-password)" \ -color=false \ ${{ parameters.template_path }} - workingDirectory: 'image-generation/packer' + workingDirectory: 'images/macos' displayName: Build image - bash: | From b89beb150d6a62d6a37997aaedb78a6faf2f27e3 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 01:57:13 +0300 Subject: [PATCH 03/17] Update structure --- .../{linux-and-win => }/download-repo.ps1 | 0 .../azure-pipelines/image-generation.yml | 6 +- .../azure-pipelines/image-generation.yml | 54 ++++++++++-------- images.CI/macos/azure-pipelines/macos1013.yml | 3 +- images.CI/macos/azure-pipelines/macos1014.yml | 3 +- images.CI/macos/azure-pipelines/macos1015.yml | 3 +- images.CI/macos/build-image.ps1 | 55 +++++++++++++++++++ images.CI/macos/destroy_vm.sh | 18 ++++++ 8 files changed, 110 insertions(+), 32 deletions(-) rename images.CI/{linux-and-win => }/download-repo.ps1 (100%) create mode 100644 images.CI/macos/build-image.ps1 create mode 100644 images.CI/macos/destroy_vm.sh diff --git a/images.CI/linux-and-win/download-repo.ps1 b/images.CI/download-repo.ps1 similarity index 100% rename from images.CI/linux-and-win/download-repo.ps1 rename to images.CI/download-repo.ps1 diff --git a/images.CI/linux-and-win/azure-pipelines/image-generation.yml b/images.CI/linux-and-win/azure-pipelines/image-generation.yml index 30499ce3f..38aabd142 100644 --- a/images.CI/linux-and-win/azure-pipelines/image-generation.yml +++ b/images.CI/linux-and-win/azure-pipelines/image-generation.yml @@ -26,7 +26,7 @@ jobs: displayName: 'Build VM' inputs: targetType: filePath - filePath: ./images.CI/build-image.ps1 + filePath: ./images.CI/linux-and-win/build-image.ps1 arguments: -ResourcesNamePrefix $(Build.BuildId) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` @@ -45,7 +45,7 @@ jobs: displayName: 'Create release for VM deployment' inputs: targetType: filePath - filePath: ./images.CI/create-release.ps1 + filePath: ./images.CI/linux-and-win/create-release.ps1 arguments: -BuildId $(Build.BuildId) ` -Organization $(RELEASE_TARGET_ORGANIZATION) ` -DefinitionId $(RELEASE_TARGET_DEFINITION_ID) ` @@ -66,7 +66,7 @@ jobs: condition: always() inputs: targetType: filePath - filePath: ./images.CI/cleanup.ps1 + filePath: ./images.CI/linux-and-win/cleanup.ps1 arguments: -ResourcesNamePrefix $(Build.BuildId) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 40cf4d43a..df1c056e2 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -28,30 +28,30 @@ jobs: SourceFolder: 'images/macos/provision/log/' RemoveSourceFolder: true - - bash: | - export PACKER_LOG=0 - packer build -on-error=abort \ - -var="vcenter_server=$(vcenter_server_v2)" \ - -var="vcenter_username=$(vcenter_username_v2)" \ - -var="vcenter_password=$(vcenter_password_v2)" \ - -var="vcenter_datacenter=$(vcenter_datacenter_v2)" \ - -var="cluster_or_esxi_host=$(esxi_cluster_v2)" \ - -var="esxi_datastore=${{ parameters.target_datastore }}" \ - -var="output_folder=mms-output" \ - -var="vm_username=$(vm_username)" \ - -var="vm_password=$(vm_password)" \ - -var="build_id=$(Build.BuildNumber)" \ - -var="baseimage_name=${{ parameters.base_image_name }}" \ - -var="azure_storage_sas=$(AZURE_STORAGE_SAS)" \ - -var="azure_storage_account=$(AZURE_STORAGE_ACCOUNT)" \ - -var="build_assets_url=$(appcenter_build_assets_url)" \ - -var="github_feed_token=$(GITHUB_FEED_TOKEN)" \ - -var="xcode_install_user=$(xcode-installation-user)" \ - -var="xcode_install_password=$(xcode-installation-password)" \ - -color=false \ - ${{ parameters.template_path }} + - task: PowerShell@2 + displayName: 'Build VM' + inputs: + targetType: filePath + filePath: ./images.CI/macos/build-image.ps1 + arguments: -Image "${{ parameters.image_name }}" ` + -VcenterServer "$(vcenter_server_v2)" ` + -VcenterUsername "$(vcenter_username_v2)" ` + -VcenterPassword "$(vcenter_password_v2)" ` + -VcenterDatacenter "$(vcenter_datacenter_v2)" ` + -EsxiCluster "$(esxi_cluster_v2)" ` + -TargetDatastore "${{ parameters.target_datastore }}" ` + -OutputFolder "mms-output" ` + -VmUsername "$(vm_username)" ` + -VmPassword "$(vm_password)" ` + -BuildId "$(Build.BuildNumber)" ` + -BaseImageName "${{ parameters.base_image_name }}" ` + -AzureStorageSas "$(AZURE_STORAGE_SAS)" ` + -AzureStorageAccount "$(AZURE_STORAGE_ACCOUNT)" ` + -BuildAssetsUrl "$(appcenter_build_assets_url)" ` + -GithubFeedToken "$(GITHUB_FEED_TOKEN)" ` + -XcodeInstallationUser "$(xcode-installation-user)" ` + -XcodeInstallationPassword "$(xcode-installation-password)" workingDirectory: 'images/macos' - displayName: Build image - bash: | echo "Copy image output files" @@ -82,3 +82,11 @@ jobs: failTaskOnFailedTests: true displayName: Publish test results condition: always() + + - task: Bash@3 + inputs: + targetType: filePath + filePath: ./images.CI/macos/destroy_vm.sh + arguments: '"$(vcenter_server_v2)" "$(vcenter_username_v2)" "$(vcenter_password_v2)" "$(vcenter_datacenter_v2)/vm/mms-output/$(Build.BuildNumber)"' + condition: eq(variables['Agent.JobStatus'], 'Canceled') + displayName: Destroy VM (if build canceled only) \ No newline at end of file diff --git a/images.CI/macos/azure-pipelines/macos1013.yml b/images.CI/macos/azure-pipelines/macos1013.yml index 04a76346d..2dfeffdeb 100644 --- a/images.CI/macos/azure-pipelines/macos1013.yml +++ b/images.CI/macos/azure-pipelines/macos1013.yml @@ -18,6 +18,5 @@ jobs: parameters: image_label: 'macOS High Sierra' base_image_name: 'clean-macOS-10.13.6-380Gb-SIPoff_runner' - template_path: 'templates/macOS-10.13.json' + image_name: 'macOS-10.13' target_datastore: 'ds-image' - stable: false diff --git a/images.CI/macos/azure-pipelines/macos1014.yml b/images.CI/macos/azure-pipelines/macos1014.yml index 8149c55c2..521f7abf9 100644 --- a/images.CI/macos/azure-pipelines/macos1014.yml +++ b/images.CI/macos/azure-pipelines/macos1014.yml @@ -18,6 +18,5 @@ jobs: parameters: image_label: 'macOS Mojave' base_image_name: 'clean-macOS-10.14-380Gb_runner' - template_path: 'templates/macOS-10.14.json' + image_name: 'macOS-10.14' target_datastore: 'ds-image' - stable: false diff --git a/images.CI/macos/azure-pipelines/macos1015.yml b/images.CI/macos/azure-pipelines/macos1015.yml index 7fc37016a..eea21f4ea 100644 --- a/images.CI/macos/azure-pipelines/macos1015.yml +++ b/images.CI/macos/azure-pipelines/macos1015.yml @@ -18,6 +18,5 @@ jobs: parameters: image_label: 'macOS Catalina' base_image_name: 'clean-macOS-10.15-380Gb-runner' - template_path: 'templates/macOS-10.15.json' + image_name: 'macOS-10.15' target_datastore: 'ds-image' - stable: false diff --git a/images.CI/macos/build-image.ps1 b/images.CI/macos/build-image.ps1 new file mode 100644 index 000000000..eae76eebe --- /dev/null +++ b/images.CI/macos/build-image.ps1 @@ -0,0 +1,55 @@ +param( + [String] [Parameter (Mandatory=$true)] $Image, + [String] [Parameter (Mandatory=$true)] $VcenterServer, + [String] [Parameter (Mandatory=$true)] $VcenterUsername, + [String] [Parameter (Mandatory=$true)] $VcenterPassword, + [String] [Parameter (Mandatory=$true)] $VcenterDatacenter, + [String] [Parameter (Mandatory=$true)] $EsxiCluster, + [String] [Parameter (Mandatory=$true)] $TargetDatastore, + [String] [Parameter (Mandatory=$true)] $OutputFolder, + [String] [Parameter (Mandatory=$true)] $VmUsername, + [String] [Parameter (Mandatory=$true)] $VmPassword, + [String] [Parameter (Mandatory=$true)] $BuildId, + [String] [Parameter (Mandatory=$true)] $BaseImageName, + [String] [Parameter (Mandatory=$true)] $AzureStorageSas, + [String] [Parameter (Mandatory=$true)] $AzureStorageAccount, + [String] [Parameter (Mandatory=$true)] $BuildAssetsUrl, + [String] [Parameter (Mandatory=$true)] $GithubFeedToken, + [String] [Parameter (Mandatory=$true)] $XcodeInstallationUser, + [String] [Parameter (Mandatory=$true)] $XcodeInstallationPassword +) + +$TemplatePath = (Get-ChildItem -Path "images" -Include "$Image.json" -Recurse -Depth 2).FullName +if (-not $TemplatePath) +{ + Write-Error "'-Image' parameter is not valid. You have to specify correct image type." + exit 1 +} + +$env:PACKER_LOG = 0 + +packer validate -syntax-only $TemplatePath + +Write-Host "Show Packer Version" +packer --version + +Write-Host "Build $Image VM" +packer build -on-error=abort -var="vcenter_server=$VcenterServer" ` + -var="vcenter_username=$VcenterUsername" ` + -var="vcenter_password=$VcenterPassword" ` + -var="vcenter_datacenter=$VcenterDatacenter" ` + -var="cluster_or_esxi_host=$EsxiCluster" ` + -var="esxi_datastore=$TargetDatastore" ` + -var="output_folder=$OutputFolder" ` + -var="vm_username=$VmUsername" ` + -var="vm_password=$VmPassword" ` + -var="build_id=$BuildId" ` + -var="baseimage_name=$BaseImageName" ` + -var="azure_storage_sas=$AzureStorageSas" ` + -var="azure_storage_account=$AzureStorageAccount" ` + -var="build_assets_url=$BuildAssetsUrl" ` + -var="github_feed_token=$GithubFeedToken" ` + -var="xcode_install_user=$XcodeInstallationUser" ` + -var="xcode_install_password=$XcodeInstallationPassword" ` + -color=false ` + $TemplatePath \ No newline at end of file diff --git a/images.CI/macos/destroy_vm.sh b/images.CI/macos/destroy_vm.sh new file mode 100644 index 000000000..cd7863cda --- /dev/null +++ b/images.CI/macos/destroy_vm.sh @@ -0,0 +1,18 @@ +VCENTER_SERVER=$1 +VCENTER_USERNAME=$2 +VCENTER_PASSWORD=$3 +VM_IPATH=$4 + +URLENCODE="image-generation/bootstrap/support/urlencode.sh" + +govc vm.power -k=true \ +-off=true \ +-u "$VCENTER_USERNAME:`$URLENCODE "$VCENTER_PASSWORD"`@$VCENTER_SERVER" \ +-vm.ipath="$VM_IPATH" + +#Time to VM's shutdown +sleep 30 + +govc vm.destroy -k=true \ +-u "$VCENTER_USERNAME:`$URLENCODE "$VCENTER_PASSWORD"`@$VCENTER_SERVER" \ +-vm.ipath="$VM_IPATH" \ No newline at end of file From 7f3dce97050097d6ea1a149a3ceba1d2074e3c2e Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 02:09:16 +0300 Subject: [PATCH 04/17] Fix build VM step --- .../azure-pipelines/image-generation.yml | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index df1c056e2..eb95bf621 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -28,29 +28,28 @@ jobs: SourceFolder: 'images/macos/provision/log/' RemoveSourceFolder: true - - task: PowerShell@2 + - pwsh: | + ./images.CI/macos/build-image.ps1 -Image "${{ parameters.image_name }}" ` + -VcenterServer "$(vcenter_server_v2)" ` + -VcenterUsername "$(vcenter_username_v2)" ` + -VcenterPassword "$(vcenter_password_v2)" ` + -VcenterDatacenter "$(vcenter_datacenter_v2)" ` + -EsxiCluster "$(esxi_cluster_v2)" ` + -TargetDatastore "${{ parameters.target_datastore }}" ` + -OutputFolder "mms-output" ` + -VmUsername "$(vm_username)" ` + -VmPassword "$(vm_password)" ` + -BuildId "$(Build.BuildNumber)" ` + -BaseImageName "${{ parameters.base_image_name }}" ` + -AzureStorageSas "$(AZURE_STORAGE_SAS)" ` + -AzureStorageAccount "$(AZURE_STORAGE_ACCOUNT)" ` + -BuildAssetsUrl "$(appcenter_build_assets_url)" ` + -GithubFeedToken "$(GITHUB_FEED_TOKEN)" ` + -XcodeInstallationUser "$(xcode-installation-user)" ` + -XcodeInstallationPassword "$(xcode-installation-password)" displayName: 'Build VM' - inputs: - targetType: filePath - filePath: ./images.CI/macos/build-image.ps1 - arguments: -Image "${{ parameters.image_name }}" ` - -VcenterServer "$(vcenter_server_v2)" ` - -VcenterUsername "$(vcenter_username_v2)" ` - -VcenterPassword "$(vcenter_password_v2)" ` - -VcenterDatacenter "$(vcenter_datacenter_v2)" ` - -EsxiCluster "$(esxi_cluster_v2)" ` - -TargetDatastore "${{ parameters.target_datastore }}" ` - -OutputFolder "mms-output" ` - -VmUsername "$(vm_username)" ` - -VmPassword "$(vm_password)" ` - -BuildId "$(Build.BuildNumber)" ` - -BaseImageName "${{ parameters.base_image_name }}" ` - -AzureStorageSas "$(AZURE_STORAGE_SAS)" ` - -AzureStorageAccount "$(AZURE_STORAGE_ACCOUNT)" ` - -BuildAssetsUrl "$(appcenter_build_assets_url)" ` - -GithubFeedToken "$(GITHUB_FEED_TOKEN)" ` - -XcodeInstallationUser "$(xcode-installation-user)" ` - -XcodeInstallationPassword "$(xcode-installation-password)" + env: + PACKER_LOG: 0 workingDirectory: 'images/macos' - bash: | From e918791a0a1529c22c5596626b2aa69bbfff4542 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 02:13:24 +0300 Subject: [PATCH 05/17] fix path to the script --- .../azure-pipelines/image-generation.yml | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index eb95bf621..4ee75821e 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -29,24 +29,24 @@ jobs: RemoveSourceFolder: true - pwsh: | - ./images.CI/macos/build-image.ps1 -Image "${{ parameters.image_name }}" ` - -VcenterServer "$(vcenter_server_v2)" ` - -VcenterUsername "$(vcenter_username_v2)" ` - -VcenterPassword "$(vcenter_password_v2)" ` - -VcenterDatacenter "$(vcenter_datacenter_v2)" ` - -EsxiCluster "$(esxi_cluster_v2)" ` - -TargetDatastore "${{ parameters.target_datastore }}" ` - -OutputFolder "mms-output" ` - -VmUsername "$(vm_username)" ` - -VmPassword "$(vm_password)" ` - -BuildId "$(Build.BuildNumber)" ` - -BaseImageName "${{ parameters.base_image_name }}" ` - -AzureStorageSas "$(AZURE_STORAGE_SAS)" ` - -AzureStorageAccount "$(AZURE_STORAGE_ACCOUNT)" ` - -BuildAssetsUrl "$(appcenter_build_assets_url)" ` - -GithubFeedToken "$(GITHUB_FEED_TOKEN)" ` - -XcodeInstallationUser "$(xcode-installation-user)" ` - -XcodeInstallationPassword "$(xcode-installation-password)" + $(build.sourcesDirectory)/images.CI/macos/build-image.ps1 -Image "${{ parameters.image_name }}" ` + -VcenterServer "$(vcenter_server_v2)" ` + -VcenterUsername "$(vcenter_username_v2)" ` + -VcenterPassword "$(vcenter_password_v2)" ` + -VcenterDatacenter "$(vcenter_datacenter_v2)" ` + -EsxiCluster "$(esxi_cluster_v2)" ` + -TargetDatastore "${{ parameters.target_datastore }}" ` + -OutputFolder "mms-output" ` + -VmUsername "$(vm_username)" ` + -VmPassword "$(vm_password)" ` + -BuildId "$(Build.BuildNumber)" ` + -BaseImageName "${{ parameters.base_image_name }}" ` + -AzureStorageSas "$(AZURE_STORAGE_SAS)" ` + -AzureStorageAccount "$(AZURE_STORAGE_ACCOUNT)" ` + -BuildAssetsUrl "$(appcenter_build_assets_url)" ` + -GithubFeedToken "$(GITHUB_FEED_TOKEN)" ` + -XcodeInstallationUser "$(xcode-installation-user)" ` + -XcodeInstallationPassword "$(xcode-installation-password)" displayName: 'Build VM' env: PACKER_LOG: 0 From 3f3e73819a4a6ccacb26b449cea739837036e4a4 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 02:15:55 +0300 Subject: [PATCH 06/17] fix build-image.ps1 script --- images.CI/macos/build-image.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images.CI/macos/build-image.ps1 b/images.CI/macos/build-image.ps1 index eae76eebe..1e462f9d4 100644 --- a/images.CI/macos/build-image.ps1 +++ b/images.CI/macos/build-image.ps1 @@ -19,7 +19,7 @@ param( [String] [Parameter (Mandatory=$true)] $XcodeInstallationPassword ) -$TemplatePath = (Get-ChildItem -Path "images" -Include "$Image.json" -Recurse -Depth 2).FullName +$TemplatePath = (Get-ChildItem -Path "templates" -Include "$Image.json" -Recurse -Depth 2).FullName if (-not $TemplatePath) { Write-Error "'-Image' parameter is not valid. You have to specify correct image type." From ab5c601f875bd87d74e5d701d97ef20f83d41584 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 02:23:17 +0300 Subject: [PATCH 07/17] Remove destroy vm script --- .../macos/azure-pipelines/image-generation.yml | 8 -------- images.CI/macos/destroy_vm.sh | 18 ------------------ 2 files changed, 26 deletions(-) delete mode 100644 images.CI/macos/destroy_vm.sh diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 4ee75821e..98aec59b8 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -81,11 +81,3 @@ jobs: failTaskOnFailedTests: true displayName: Publish test results condition: always() - - - task: Bash@3 - inputs: - targetType: filePath - filePath: ./images.CI/macos/destroy_vm.sh - arguments: '"$(vcenter_server_v2)" "$(vcenter_username_v2)" "$(vcenter_password_v2)" "$(vcenter_datacenter_v2)/vm/mms-output/$(Build.BuildNumber)"' - condition: eq(variables['Agent.JobStatus'], 'Canceled') - displayName: Destroy VM (if build canceled only) \ No newline at end of file diff --git a/images.CI/macos/destroy_vm.sh b/images.CI/macos/destroy_vm.sh deleted file mode 100644 index cd7863cda..000000000 --- a/images.CI/macos/destroy_vm.sh +++ /dev/null @@ -1,18 +0,0 @@ -VCENTER_SERVER=$1 -VCENTER_USERNAME=$2 -VCENTER_PASSWORD=$3 -VM_IPATH=$4 - -URLENCODE="image-generation/bootstrap/support/urlencode.sh" - -govc vm.power -k=true \ --off=true \ --u "$VCENTER_USERNAME:`$URLENCODE "$VCENTER_PASSWORD"`@$VCENTER_SERVER" \ --vm.ipath="$VM_IPATH" - -#Time to VM's shutdown -sleep 30 - -govc vm.destroy -k=true \ --u "$VCENTER_USERNAME:`$URLENCODE "$VCENTER_PASSWORD"`@$VCENTER_SERVER" \ --vm.ipath="$VM_IPATH" \ No newline at end of file From 7e0e8fcab61ca9c482d8d605c3fc1353b939c1a5 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 15:44:26 +0300 Subject: [PATCH 08/17] Fix structure --- .../azure-pipelines/image-generation.yml | 41 +++++++------- images.CI/macos/azure-pipelines/macos1013.yml | 2 +- images.CI/macos/azure-pipelines/macos1014.yml | 2 +- images.CI/macos/azure-pipelines/macos1015.yml | 2 +- images.CI/macos/azure-pipelines/macos110.yml | 22 ++++++++ images.CI/macos/build-image.ps1 | 55 ------------------- 6 files changed, 45 insertions(+), 79 deletions(-) create mode 100644 images.CI/macos/azure-pipelines/macos110.yml delete mode 100644 images.CI/macos/build-image.ps1 diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 98aec59b8..bd39df574 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -28,25 +28,24 @@ jobs: SourceFolder: 'images/macos/provision/log/' RemoveSourceFolder: true - - pwsh: | - $(build.sourcesDirectory)/images.CI/macos/build-image.ps1 -Image "${{ parameters.image_name }}" ` - -VcenterServer "$(vcenter_server_v2)" ` - -VcenterUsername "$(vcenter_username_v2)" ` - -VcenterPassword "$(vcenter_password_v2)" ` - -VcenterDatacenter "$(vcenter_datacenter_v2)" ` - -EsxiCluster "$(esxi_cluster_v2)" ` - -TargetDatastore "${{ parameters.target_datastore }}" ` - -OutputFolder "mms-output" ` - -VmUsername "$(vm_username)" ` - -VmPassword "$(vm_password)" ` - -BuildId "$(Build.BuildNumber)" ` - -BaseImageName "${{ parameters.base_image_name }}" ` - -AzureStorageSas "$(AZURE_STORAGE_SAS)" ` - -AzureStorageAccount "$(AZURE_STORAGE_ACCOUNT)" ` - -BuildAssetsUrl "$(appcenter_build_assets_url)" ` - -GithubFeedToken "$(GITHUB_FEED_TOKEN)" ` - -XcodeInstallationUser "$(xcode-installation-user)" ` - -XcodeInstallationPassword "$(xcode-installation-password)" + - bash: | + packer build -on-error=abort \ + -var="vcenter_server=$(vcenter_server_v2)" \ + -var="vcenter_username=$(vcenter_username_v2)" \ + -var="vcenter_password=$(vcenter_password_v2)" \ + -var="vcenter_datacenter=$(vcenter_datacenter_v2)" \ + -var="cluster_or_esxi_host=$(esxi_cluster_v2)" \ + -var="esxi_datastore=${{ parameters.target_datastore }}" \ + -var="output_folder=mms-output" \ + -var="vm_username=$(vm_username)" \ + -var="vm_password=$(vm_password)" \ + -var="build_id=$(Build.BuildNumber)" \ + -var="baseimage_name=${{ parameters.base_image_name }}" \ + -var="github_feed_token=$(GITHUB_FEED_TOKEN)" \ + -var="xcode_install_user=$(xcode-installation-user)" \ + -var="xcode_install_password=$(xcode-installation-password)" \ + -color=false \ + ${{ parameters.template_path }} displayName: 'Build VM' env: PACKER_LOG: 0 @@ -54,10 +53,10 @@ jobs: - bash: | echo "Copy image output files" - cp -R "image-generation/image-output/software-report/." "$(Build.ArtifactStagingDirectory)/" + cp -R "images/image-output/software-report/." "$(Build.ArtifactStagingDirectory)/" echo "Copy test results" - cp -R "image-generation/image-output/tests/." "$(Common.TestResultsDirectory)/" + cp -R "images/image-output/tests/." "$(Common.TestResultsDirectory)/" ls $(Common.TestResultsDirectory) echo "Put VM name to 'VM_Done_Name' file" diff --git a/images.CI/macos/azure-pipelines/macos1013.yml b/images.CI/macos/azure-pipelines/macos1013.yml index 2dfeffdeb..d214bccf6 100644 --- a/images.CI/macos/azure-pipelines/macos1013.yml +++ b/images.CI/macos/azure-pipelines/macos1013.yml @@ -18,5 +18,5 @@ jobs: parameters: image_label: 'macOS High Sierra' base_image_name: 'clean-macOS-10.13.6-380Gb-SIPoff_runner' - image_name: 'macOS-10.13' + template_path: 'templates/macOS-10.13.json' target_datastore: 'ds-image' diff --git a/images.CI/macos/azure-pipelines/macos1014.yml b/images.CI/macos/azure-pipelines/macos1014.yml index 521f7abf9..9ee818eb7 100644 --- a/images.CI/macos/azure-pipelines/macos1014.yml +++ b/images.CI/macos/azure-pipelines/macos1014.yml @@ -18,5 +18,5 @@ jobs: parameters: image_label: 'macOS Mojave' base_image_name: 'clean-macOS-10.14-380Gb_runner' - image_name: 'macOS-10.14' + template_path: 'templates/macOS-10.14.json' target_datastore: 'ds-image' diff --git a/images.CI/macos/azure-pipelines/macos1015.yml b/images.CI/macos/azure-pipelines/macos1015.yml index eea21f4ea..967f741b9 100644 --- a/images.CI/macos/azure-pipelines/macos1015.yml +++ b/images.CI/macos/azure-pipelines/macos1015.yml @@ -18,5 +18,5 @@ jobs: parameters: image_label: 'macOS Catalina' base_image_name: 'clean-macOS-10.15-380Gb-runner' - image_name: 'macOS-10.15' + template_path: 'templates/macOS-10.15.json' target_datastore: 'ds-image' diff --git a/images.CI/macos/azure-pipelines/macos110.yml b/images.CI/macos/azure-pipelines/macos110.yml new file mode 100644 index 000000000..7a7864270 --- /dev/null +++ b/images.CI/macos/azure-pipelines/macos110.yml @@ -0,0 +1,22 @@ +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - main + always: true + +trigger: none +pr: + autoCancel: true + branches: + include: + - main + +jobs: +- template: image-generation-job.yml + parameters: + image_label: 'macOS Big Sur' + base_image_name: 'clean-macOS-11.0-380Gb-runner' + template_path: 'templates/macOS-11.0.json' + target_datastore: 'ds-image' diff --git a/images.CI/macos/build-image.ps1 b/images.CI/macos/build-image.ps1 deleted file mode 100644 index 1e462f9d4..000000000 --- a/images.CI/macos/build-image.ps1 +++ /dev/null @@ -1,55 +0,0 @@ -param( - [String] [Parameter (Mandatory=$true)] $Image, - [String] [Parameter (Mandatory=$true)] $VcenterServer, - [String] [Parameter (Mandatory=$true)] $VcenterUsername, - [String] [Parameter (Mandatory=$true)] $VcenterPassword, - [String] [Parameter (Mandatory=$true)] $VcenterDatacenter, - [String] [Parameter (Mandatory=$true)] $EsxiCluster, - [String] [Parameter (Mandatory=$true)] $TargetDatastore, - [String] [Parameter (Mandatory=$true)] $OutputFolder, - [String] [Parameter (Mandatory=$true)] $VmUsername, - [String] [Parameter (Mandatory=$true)] $VmPassword, - [String] [Parameter (Mandatory=$true)] $BuildId, - [String] [Parameter (Mandatory=$true)] $BaseImageName, - [String] [Parameter (Mandatory=$true)] $AzureStorageSas, - [String] [Parameter (Mandatory=$true)] $AzureStorageAccount, - [String] [Parameter (Mandatory=$true)] $BuildAssetsUrl, - [String] [Parameter (Mandatory=$true)] $GithubFeedToken, - [String] [Parameter (Mandatory=$true)] $XcodeInstallationUser, - [String] [Parameter (Mandatory=$true)] $XcodeInstallationPassword -) - -$TemplatePath = (Get-ChildItem -Path "templates" -Include "$Image.json" -Recurse -Depth 2).FullName -if (-not $TemplatePath) -{ - Write-Error "'-Image' parameter is not valid. You have to specify correct image type." - exit 1 -} - -$env:PACKER_LOG = 0 - -packer validate -syntax-only $TemplatePath - -Write-Host "Show Packer Version" -packer --version - -Write-Host "Build $Image VM" -packer build -on-error=abort -var="vcenter_server=$VcenterServer" ` - -var="vcenter_username=$VcenterUsername" ` - -var="vcenter_password=$VcenterPassword" ` - -var="vcenter_datacenter=$VcenterDatacenter" ` - -var="cluster_or_esxi_host=$EsxiCluster" ` - -var="esxi_datastore=$TargetDatastore" ` - -var="output_folder=$OutputFolder" ` - -var="vm_username=$VmUsername" ` - -var="vm_password=$VmPassword" ` - -var="build_id=$BuildId" ` - -var="baseimage_name=$BaseImageName" ` - -var="azure_storage_sas=$AzureStorageSas" ` - -var="azure_storage_account=$AzureStorageAccount" ` - -var="build_assets_url=$BuildAssetsUrl" ` - -var="github_feed_token=$GithubFeedToken" ` - -var="xcode_install_user=$XcodeInstallationUser" ` - -var="xcode_install_password=$XcodeInstallationPassword" ` - -color=false ` - $TemplatePath \ No newline at end of file From eff7f1d78339cfcab4e79be3f5ccae63142ef996 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 16:46:44 +0300 Subject: [PATCH 09/17] Add name to ymls and add output_folder variable --- images.CI/macos/azure-pipelines/image-generation.yml | 2 +- images.CI/macos/azure-pipelines/macos1013.yml | 1 + images.CI/macos/azure-pipelines/macos1014.yml | 1 + images.CI/macos/azure-pipelines/macos1015.yml | 1 + images.CI/macos/azure-pipelines/macos110.yml | 3 ++- 5 files changed, 6 insertions(+), 2 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index bd39df574..345656951 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -36,7 +36,7 @@ jobs: -var="vcenter_datacenter=$(vcenter_datacenter_v2)" \ -var="cluster_or_esxi_host=$(esxi_cluster_v2)" \ -var="esxi_datastore=${{ parameters.target_datastore }}" \ - -var="output_folder=mms-output" \ + -var="output_folder=$(output_folder)" \ -var="vm_username=$(vm_username)" \ -var="vm_password=$(vm_password)" \ -var="build_id=$(Build.BuildNumber)" \ diff --git a/images.CI/macos/azure-pipelines/macos1013.yml b/images.CI/macos/azure-pipelines/macos1013.yml index d214bccf6..a7b9a4a19 100644 --- a/images.CI/macos/azure-pipelines/macos1013.yml +++ b/images.CI/macos/azure-pipelines/macos1013.yml @@ -1,3 +1,4 @@ +name: macOS-10.13_$(date:yyyyMMdd)$(rev:.r)_unstable schedules: - cron: "0 0 * * *" displayName: Daily diff --git a/images.CI/macos/azure-pipelines/macos1014.yml b/images.CI/macos/azure-pipelines/macos1014.yml index 9ee818eb7..21f7e48c1 100644 --- a/images.CI/macos/azure-pipelines/macos1014.yml +++ b/images.CI/macos/azure-pipelines/macos1014.yml @@ -1,3 +1,4 @@ +name: macOS-10.14_$(date:yyyyMMdd)$(rev:.r)_unstable schedules: - cron: "0 0 * * *" displayName: Daily diff --git a/images.CI/macos/azure-pipelines/macos1015.yml b/images.CI/macos/azure-pipelines/macos1015.yml index 967f741b9..f42980cc4 100644 --- a/images.CI/macos/azure-pipelines/macos1015.yml +++ b/images.CI/macos/azure-pipelines/macos1015.yml @@ -1,3 +1,4 @@ +name: macOS-10.15_$(date:yyyyMMdd)$(rev:.r)_unstable schedules: - cron: "0 0 * * *" displayName: Daily diff --git a/images.CI/macos/azure-pipelines/macos110.yml b/images.CI/macos/azure-pipelines/macos110.yml index 7a7864270..ada6dcfe1 100644 --- a/images.CI/macos/azure-pipelines/macos110.yml +++ b/images.CI/macos/azure-pipelines/macos110.yml @@ -1,3 +1,4 @@ +name: macOS-11.0_$(date:yyyyMMdd)$(rev:.r)_unstable schedules: - cron: "0 0 * * *" displayName: Daily @@ -14,7 +15,7 @@ pr: - main jobs: -- template: image-generation-job.yml +- template: image-generation.yml parameters: image_label: 'macOS Big Sur' base_image_name: 'clean-macOS-11.0-380Gb-runner' From 221a58cd32dfa66e8cf21e854d4605eb4d61f8d7 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 22:06:19 +0300 Subject: [PATCH 10/17] Remove sensitive data from logs --- .../macos/azure-pipelines/image-generation.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 345656951..8236a7b7c 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -28,7 +28,12 @@ jobs: SourceFolder: 'images/macos/provision/log/' RemoveSourceFolder: true - - bash: | + - pwsh: | + $SensitiveData = @( + 'IP address:', + 'Using ssh communicator to connect:' + ) + packer build -on-error=abort \ -var="vcenter_server=$(vcenter_server_v2)" \ -var="vcenter_username=$(vcenter_username_v2)" \ @@ -45,7 +50,13 @@ jobs: -var="xcode_install_user=$(xcode-installation-user)" \ -var="xcode_install_password=$(xcode-installation-password)" \ -color=false \ - ${{ parameters.template_path }} + ${{ parameters.template_path }} ` + | Where-Object { + #Filter sensitive data from Packer logs + $currentString = $_ + $sensitiveString = $SensitiveData | Where-Object { $currentString -match $_ } + $sensitiveString -eq $null + } displayName: 'Build VM' env: PACKER_LOG: 0 From d166ca23b751942b207429518c4c0930a51dc49f Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 22:11:50 +0300 Subject: [PATCH 11/17] fix typo --- .../azure-pipelines/image-generation.yml | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 8236a7b7c..8eb3f6998 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -34,22 +34,22 @@ jobs: 'Using ssh communicator to connect:' ) - packer build -on-error=abort \ - -var="vcenter_server=$(vcenter_server_v2)" \ - -var="vcenter_username=$(vcenter_username_v2)" \ - -var="vcenter_password=$(vcenter_password_v2)" \ - -var="vcenter_datacenter=$(vcenter_datacenter_v2)" \ - -var="cluster_or_esxi_host=$(esxi_cluster_v2)" \ - -var="esxi_datastore=${{ parameters.target_datastore }}" \ - -var="output_folder=$(output_folder)" \ - -var="vm_username=$(vm_username)" \ - -var="vm_password=$(vm_password)" \ - -var="build_id=$(Build.BuildNumber)" \ - -var="baseimage_name=${{ parameters.base_image_name }}" \ - -var="github_feed_token=$(GITHUB_FEED_TOKEN)" \ - -var="xcode_install_user=$(xcode-installation-user)" \ - -var="xcode_install_password=$(xcode-installation-password)" \ - -color=false \ + packer build -on-error=abort ` + -var="vcenter_server=$(vcenter_server_v2)" ` + -var="vcenter_username=$(vcenter_username_v2)" ` + -var="vcenter_password=$(vcenter_password_v2)" ` + -var="vcenter_datacenter=$(vcenter_datacenter_v2)" ` + -var="cluster_or_esxi_host=$(esxi_cluster_v2)" ` + -var="esxi_datastore=${{ parameters.target_datastore }}" ` + -var="output_folder=$(output_folder)" ` + -var="vm_username=$(vm_username)" ` + -var="vm_password=$(vm_password)" ` + -var="build_id=$(Build.BuildNumber)" ` + -var="baseimage_name=${{ parameters.base_image_name }}" ` + -var="github_feed_token=$(GITHUB_FEED_TOKEN)" ` + -var="xcode_install_user=$(xcode-installation-user)" ` + -var="xcode_install_password=$(xcode-installation-password)" ` + -color=false ` ${{ parameters.template_path }} ` | Where-Object { #Filter sensitive data from Packer logs From 9def9a1f02ace2f08543975f204522dd3139d56b Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 15 Sep 2020 15:25:32 +0300 Subject: [PATCH 12/17] Add "validate contributor permissions" step --- images.CI/macos/azure-pipelines/image-generation.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 8eb3f6998..5e42236c9 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -9,6 +9,17 @@ jobs: - group: Mac-Cloud Image Generation Key Vault steps: + - pwsh: | + $validСontributor = ${$env:CONTRIBUTOR_ALLOWLIST}.Split(",") | Where-Object { $_ -eq $env:BUILD_SOURCEVERSIONAUTHOR } ` + | Select-Object -First 1 + + if (-not $validСontributor) { + Write-Host "Failed to start this build. $env:BUILD_SOURCEVERSIONAUTHOR is an unknown contributor" + Write-Host "Please add $env:BUILD_SOURCEVERSIONAUTHOR to the allowed list to run this build" + exit 1 + } + displayName: Validate contributor permissions + - checkout: self clean: true fetchDepth: 1 From 80d5ed8f2a2e1159fccaf0386ee8d55a5f3e2963 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 15 Sep 2020 15:37:18 +0300 Subject: [PATCH 13/17] Add trim --- images.CI/macos/azure-pipelines/image-generation.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 5e42236c9..e453bf6f8 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -10,12 +10,12 @@ jobs: steps: - pwsh: | - $validСontributor = ${$env:CONTRIBUTOR_ALLOWLIST}.Split(",") | Where-Object { $_ -eq $env:BUILD_SOURCEVERSIONAUTHOR } ` - | Select-Object -First 1 + $validСontributor = ${$env:CONTRIBUTOR_ALLOWLIST}.Split(",").Trim() | Where-Object { $_ -eq $env:BUILD_SOURCEVERSIONAUTHOR } ` + | Select-Object -First 1 if (-not $validСontributor) { Write-Host "Failed to start this build. $env:BUILD_SOURCEVERSIONAUTHOR is an unknown contributor" - Write-Host "Please add $env:BUILD_SOURCEVERSIONAUTHOR to the allowed list to run this build" + Write-Host "Please add $env:BUILD_SOURCEVERSIONAUTHOR to the allowed list to run builds" exit 1 } displayName: Validate contributor permissions From 0b2ee087e36fdf165bc7c062d288953a06974845 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 15 Sep 2020 16:10:29 +0300 Subject: [PATCH 14/17] Fix comments --- images.CI/macos/azure-pipelines/image-generation.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index e453bf6f8..b650fe0ae 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -3,15 +3,16 @@ jobs: displayName: Image Generation (${{ parameters.image_label }}) timeoutInMinutes: 720 pool: - name: Mac-Cloud V2 Image Generation + name: Mac-Cloud Image Generation variables: - group: Mac-Cloud Image Generation - group: Mac-Cloud Image Generation Key Vault steps: - pwsh: | - $validСontributor = ${$env:CONTRIBUTOR_ALLOWLIST}.Split(",").Trim() | Where-Object { $_ -eq $env:BUILD_SOURCEVERSIONAUTHOR } ` - | Select-Object -First 1 + $allowedContributors = $env:CONTRIBUTOR_ALLOWLIST.Split(",").Trim() + $validСontributor = $allowedContributors | Where-Object { $_ -eq $env:BUILD_SOURCEVERSIONAUTHOR } ` + | Select-Object -First 1 if (-not $validСontributor) { Write-Host "Failed to start this build. $env:BUILD_SOURCEVERSIONAUTHOR is an unknown contributor" From 734baaa941acb59a0e6526aef4f32da8142c9b47 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 15 Sep 2020 19:14:07 +0300 Subject: [PATCH 15/17] Move credscan-exclusions file --- images.CI/{linux-and-win => }/credscan-exclusions.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename images.CI/{linux-and-win => }/credscan-exclusions.json (100%) diff --git a/images.CI/linux-and-win/credscan-exclusions.json b/images.CI/credscan-exclusions.json similarity index 100% rename from images.CI/linux-and-win/credscan-exclusions.json rename to images.CI/credscan-exclusions.json From 3d46b4867e4d0533d78ab271e22a02105ba0ec10 Mon Sep 17 00:00:00 2001 From: MaksimmZZZhukoff Date: Wed, 16 Sep 2020 13:27:13 +0300 Subject: [PATCH 16/17] Add script to validate contributor permissions --- .../azure-pipelines/image-generation.yml | 24 +++---- images.CI/macos/validate-contributor.ps1 | 63 +++++++++++++++++++ 2 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 images.CI/macos/validate-contributor.ps1 diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index b650fe0ae..5c2f562f1 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -9,22 +9,22 @@ jobs: - group: Mac-Cloud Image Generation Key Vault steps: - - pwsh: | - $allowedContributors = $env:CONTRIBUTOR_ALLOWLIST.Split(",").Trim() - $validСontributor = $allowedContributors | Where-Object { $_ -eq $env:BUILD_SOURCEVERSIONAUTHOR } ` - | Select-Object -First 1 - - if (-not $validСontributor) { - Write-Host "Failed to start this build. $env:BUILD_SOURCEVERSIONAUTHOR is an unknown contributor" - Write-Host "Please add $env:BUILD_SOURCEVERSIONAUTHOR to the allowed list to run builds" - exit 1 - } - displayName: Validate contributor permissions - - checkout: self clean: true fetchDepth: 1 + - task: PowerShell@2 + displayName: 'Validate contributor permissions' + condition: startsWith(variables['Build.SourceBranch'], 'refs/pull/') + inputs: + targetType: 'filePath' + filePath: ./images.CI/macos/validate-contributor.ps1 + pwsh: true + arguments: -RepositoryName "$(Build.Repository.Name)" ` + -AccessToken "$(GITHUB_FEED_TOKEN)" ` + -SourceBranch "$(Build.SourceBranch)" ` + -ContributorAllowList "$(CONTRIBUTOR_ALLOWLIST)" + - task: PowerShell@2 displayName: 'Download custom repository' condition: and(ne(variables['CUSTOM_REPOSITORY_URL'], ''), ne(variables['CUSTOM_REPOSITORY_BRANCH'], '')) diff --git a/images.CI/macos/validate-contributor.ps1 b/images.CI/macos/validate-contributor.ps1 new file mode 100644 index 000000000..312a9ef7e --- /dev/null +++ b/images.CI/macos/validate-contributor.ps1 @@ -0,0 +1,63 @@ +param( + [Parameter(Mandatory)] [string] $RepositoryName, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [string] $SourceBranch, + [Parameter(Mandatory)] [string] $ContributorAllowList +) + +function Build-AuthHeader { + param( + [Parameter(Mandatory)] [string] $AccessToken + ) + + $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("'':${AccessToken}")) + return "Basic ${base64AuthInfo}" +} + +function Get-PullRequest { + param( + [Parameter(Mandatory)] [string] $RepositoryName, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [UInt32] $PullRequestNumber + ) + + $requestUrl = "https://api.github.com/repos/$RepositoryName/pulls/$PullRequestNumber" + $authHeader = Build-AuthHeader -AccessToken $AccessToken + + $params = @{ + Method = "GET" + ContentType = "application/json" + Uri = $requestUrl + Headers = @{ Authorization = $authHeader } + } + + return Invoke-RestMethod @params +} + +function Validate-ContributorPermissions { + param( + [Parameter(Mandatory)] [string] $ContributorAllowList, + [Parameter(Mandatory)] [string] $ContributorName + ) + + $allowedContributors = $ContributorAllowList.Split(",").Trim() + $validСontributor = $allowedContributors | Where-Object { $_ -eq $ContributorName } ` + | Select-Object -First 1 + + if (-not $validСontributor) { + Write-Host "Failed to start this build. '$ContributorName' is an unknown contributor" + Write-Host "Please add '$ContributorName' to the allowed list to run builds" + exit 1 + } +} + +$pullRequestNumber = $SourceBranch.Split("/")[2] + +$pullRequestInfo = Get-PullRequest -RepositoryName $RepositoryName ` + -AccessToken $AccessToken ` + -PullRequestNumber $pullRequestNumber + +$contributorName = $pullRequestInfo.user.login + +Validate-ContributorPermissions -ContributorAllowList $ContributorAllowList ` + -ContributorName $contributorName \ No newline at end of file From 5dc70e7d03f6be2d8c3883e49f30d1c766d3f8a8 Mon Sep 17 00:00:00 2001 From: MaksimmZZZhukoff Date: Wed, 16 Sep 2020 14:36:05 +0300 Subject: [PATCH 17/17] Update variables in image-generation file --- .../azure-pipelines/image-generation.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 5c2f562f1..490cbda6d 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -21,7 +21,7 @@ jobs: filePath: ./images.CI/macos/validate-contributor.ps1 pwsh: true arguments: -RepositoryName "$(Build.Repository.Name)" ` - -AccessToken "$(GITHUB_FEED_TOKEN)" ` + -AccessToken "$(github-feed-token)" ` -SourceBranch "$(Build.SourceBranch)" ` -ContributorAllowList "$(CONTRIBUTOR_ALLOWLIST)" @@ -47,18 +47,18 @@ jobs: ) packer build -on-error=abort ` - -var="vcenter_server=$(vcenter_server_v2)" ` - -var="vcenter_username=$(vcenter_username_v2)" ` - -var="vcenter_password=$(vcenter_password_v2)" ` - -var="vcenter_datacenter=$(vcenter_datacenter_v2)" ` - -var="cluster_or_esxi_host=$(esxi_cluster_v2)" ` + -var="vcenter_server=$(vcenter-server-v2)" ` + -var="vcenter_username=$(vcenter-username-v2)" ` + -var="vcenter_password=$(vcenter-password-v2)" ` + -var="vcenter_datacenter=$(vcenter-datacenter-v2)" ` + -var="cluster_or_esxi_host=$(esxi-cluster-v2)" ` -var="esxi_datastore=${{ parameters.target_datastore }}" ` - -var="output_folder=$(output_folder)" ` - -var="vm_username=$(vm_username)" ` - -var="vm_password=$(vm_password)" ` + -var="output_folder=$(output-folder)" ` + -var="vm_username=$(vm-username)" ` + -var="vm_password=$(vm-password)" ` -var="build_id=$(Build.BuildNumber)" ` -var="baseimage_name=${{ parameters.base_image_name }}" ` - -var="github_feed_token=$(GITHUB_FEED_TOKEN)" ` + -var="github_feed_token=$(github-feed-token)" ` -var="xcode_install_user=$(xcode-installation-user)" ` -var="xcode_install_password=$(xcode-installation-password)" ` -color=false `