From 962aa94f3ad04745ba011c20abcb5a4adac09014 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Fri, 11 Sep 2020 17:34:56 +0300 Subject: [PATCH 01/31] Add CI for macos --- .../azure-pipelines/image-generation.yml | 2 +- .../azure-pipelines/ubuntu1604.yml | 0 .../azure-pipelines/ubuntu1804.yml | 0 .../azure-pipelines/ubuntu2004.yml | 0 .../azure-pipelines/windows2016.yml | 0 .../azure-pipelines/windows2019.yml | 0 images.CI/{ => linux-and-win}/build-image.ps1 | 0 images.CI/{ => linux-and-win}/cleanup.ps1 | 0 .../{ => linux-and-win}/create-release.ps1 | 0 .../credscan-exclusions.json | 0 .../{ => linux-and-win}/download-repo.ps1 | 0 .../azure-pipelines/image-generation.yml | 84 +++++++++++++++++++ images.CI/macos/azure-pipelines/macos1013.yml | 23 +++++ images.CI/macos/azure-pipelines/macos1014.yml | 23 +++++ images.CI/macos/azure-pipelines/macos1015.yml | 23 +++++ 15 files changed, 154 insertions(+), 1 deletion(-) rename images.CI/{ => linux-and-win}/azure-pipelines/image-generation.yml (98%) rename images.CI/{ => linux-and-win}/azure-pipelines/ubuntu1604.yml (100%) rename images.CI/{ => linux-and-win}/azure-pipelines/ubuntu1804.yml (100%) rename images.CI/{ => linux-and-win}/azure-pipelines/ubuntu2004.yml (100%) rename images.CI/{ => linux-and-win}/azure-pipelines/windows2016.yml (100%) rename images.CI/{ => linux-and-win}/azure-pipelines/windows2019.yml (100%) rename images.CI/{ => linux-and-win}/build-image.ps1 (100%) rename images.CI/{ => linux-and-win}/cleanup.ps1 (100%) rename images.CI/{ => linux-and-win}/create-release.ps1 (100%) rename images.CI/{ => linux-and-win}/credscan-exclusions.json (100%) rename images.CI/{ => linux-and-win}/download-repo.ps1 (100%) create mode 100644 images.CI/macos/azure-pipelines/image-generation.yml create mode 100644 images.CI/macos/azure-pipelines/macos1013.yml create mode 100644 images.CI/macos/azure-pipelines/macos1014.yml create mode 100644 images.CI/macos/azure-pipelines/macos1015.yml diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/linux-and-win/azure-pipelines/image-generation.yml similarity index 98% rename from images.CI/azure-pipelines/image-generation.yml rename to images.CI/linux-and-win/azure-pipelines/image-generation.yml index 70db48f1c..30499ce3f 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/linux-and-win/azure-pipelines/image-generation.yml @@ -20,7 +20,7 @@ jobs: targetType: 'filePath' filePath: ./images.CI/download-repo.ps1 arguments: -RepoUrl $(CUSTOM_REPOSITORY_URL) ` - -RepoBranch $(CUSTOM_REPOSITORY_BRANCH) + -RepoBranch $(CUSTOM_REPOSITORY_BRANCH) - task: PowerShell@2 displayName: 'Build VM' diff --git a/images.CI/azure-pipelines/ubuntu1604.yml b/images.CI/linux-and-win/azure-pipelines/ubuntu1604.yml similarity index 100% rename from images.CI/azure-pipelines/ubuntu1604.yml rename to images.CI/linux-and-win/azure-pipelines/ubuntu1604.yml diff --git a/images.CI/azure-pipelines/ubuntu1804.yml b/images.CI/linux-and-win/azure-pipelines/ubuntu1804.yml similarity index 100% rename from images.CI/azure-pipelines/ubuntu1804.yml rename to images.CI/linux-and-win/azure-pipelines/ubuntu1804.yml diff --git a/images.CI/azure-pipelines/ubuntu2004.yml b/images.CI/linux-and-win/azure-pipelines/ubuntu2004.yml similarity index 100% rename from images.CI/azure-pipelines/ubuntu2004.yml rename to images.CI/linux-and-win/azure-pipelines/ubuntu2004.yml diff --git a/images.CI/azure-pipelines/windows2016.yml b/images.CI/linux-and-win/azure-pipelines/windows2016.yml similarity index 100% rename from images.CI/azure-pipelines/windows2016.yml rename to images.CI/linux-and-win/azure-pipelines/windows2016.yml diff --git a/images.CI/azure-pipelines/windows2019.yml b/images.CI/linux-and-win/azure-pipelines/windows2019.yml similarity index 100% rename from images.CI/azure-pipelines/windows2019.yml rename to images.CI/linux-and-win/azure-pipelines/windows2019.yml diff --git a/images.CI/build-image.ps1 b/images.CI/linux-and-win/build-image.ps1 similarity index 100% rename from images.CI/build-image.ps1 rename to images.CI/linux-and-win/build-image.ps1 diff --git a/images.CI/cleanup.ps1 b/images.CI/linux-and-win/cleanup.ps1 similarity index 100% rename from images.CI/cleanup.ps1 rename to images.CI/linux-and-win/cleanup.ps1 diff --git a/images.CI/create-release.ps1 b/images.CI/linux-and-win/create-release.ps1 similarity index 100% rename from images.CI/create-release.ps1 rename to images.CI/linux-and-win/create-release.ps1 diff --git a/images.CI/credscan-exclusions.json b/images.CI/linux-and-win/credscan-exclusions.json similarity index 100% rename from images.CI/credscan-exclusions.json rename to images.CI/linux-and-win/credscan-exclusions.json diff --git a/images.CI/download-repo.ps1 b/images.CI/linux-and-win/download-repo.ps1 similarity index 100% rename from images.CI/download-repo.ps1 rename to images.CI/linux-and-win/download-repo.ps1 diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml new file mode 100644 index 000000000..b4af41203 --- /dev/null +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -0,0 +1,84 @@ +jobs: +- job: Image_generation + displayName: Image Generation (${{ parameters.image_label }}) + timeoutInMinutes: 720 + pool: + name: Mac-Cloud V2 Image Generation + variables: + - group: Mac-Cloud Image Generation + - group: Mac-Cloud Image Generation Key Vault + + steps: + - checkout: self + clean: true + fetchDepth: 1 + + - task: PowerShell@2 + displayName: 'Download custom repository' + condition: and(ne(variables['CUSTOM_REPOSITORY_URL'], ''), ne(variables['CUSTOM_REPOSITORY_BRANCH'], '')) + inputs: + targetType: 'filePath' + filePath: ./images.CI/download-repo.ps1 + arguments: -RepoUrl $(CUSTOM_REPOSITORY_URL) ` + -RepoBranch $(CUSTOM_REPOSITORY_BRANCH) + + - task: DeleteFiles@1 + displayName: Clean up self-hosted machine + inputs: + SourceFolder: 'image-generation/packer/provision/log/' + RemoveSourceFolder: true + + - bash: | + export PACKER_LOG=0 + packer build -on-error=abort \ + -var="vcenter_server=$(vcenter_server_v2)" \ + -var="vcenter_username=$(vcenter_username_v2)" \ + -var="vcenter_password=$(vcenter_password_v2)" \ + -var="vcenter_datacenter=$(vcenter_datacenter_v2)" \ + -var="cluster_or_esxi_host=$(esxi_cluster_v2)" \ + -var="esxi_datastore=${{ parameters.target_datastore }}" \ + -var="output_folder=mms-output" \ + -var="vm_username=$(vm_username)" \ + -var="vm_password=$(vm_password)" \ + -var="build_id=$(Build.BuildNumber)" \ + -var="baseimage_name=${{ parameters.base_image_name }}" \ + -var="azure_storage_sas=$(AZURE_STORAGE_SAS)" \ + -var="azure_storage_account=$(AZURE_STORAGE_ACCOUNT)" \ + -var="build_assets_url=$(appcenter_build_assets_url)" \ + -var="github_feed_token=$(GITHUB_FEED_TOKEN)" \ + -var="xcode_install_user=$(xcode-installation-user)" \ + -var="xcode_install_password=$(xcode-installation-password)" \ + -color=false \ + ${{ parameters.template_path }} + workingDirectory: 'image-generation/packer' + displayName: Build image + + - bash: | + echo "Copy image output files" + cp -R "image-generation/image-output/software-report/." "$(Build.ArtifactStagingDirectory)/" + + echo "Copy test results" + cp -R "image-generation/image-output/tests/." "$(Common.TestResultsDirectory)/" + ls $(Common.TestResultsDirectory) + + echo "Put VM name to 'VM_Done_Name' file" + echo "$(Build.BuildNumber)" > "$(Build.ArtifactStagingDirectory)/VM_Done_Name" + displayName: Prepare artifact + + - bash: | + cat "$(Build.ArtifactStagingDirectory)/systeminfo.md" + displayName: Print software report + + - task: PublishBuildArtifacts@1 + inputs: + ArtifactName: 'Built_VM_Artifacts' + displayName: Publish Artifacts + + - task: PublishTestResults@2 + inputs: + testResultsFiles: '*.xml' + testResultsFormat: NUnit + searchFolder: '$(Common.TestResultsDirectory)' + failTaskOnFailedTests: true + displayName: Publish test results + condition: always() diff --git a/images.CI/macos/azure-pipelines/macos1013.yml b/images.CI/macos/azure-pipelines/macos1013.yml new file mode 100644 index 000000000..04a76346d --- /dev/null +++ b/images.CI/macos/azure-pipelines/macos1013.yml @@ -0,0 +1,23 @@ +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - main + always: true + +trigger: none +pr: + autoCancel: true + branches: + include: + - main + +jobs: +- template: image-generation.yml + parameters: + image_label: 'macOS High Sierra' + base_image_name: 'clean-macOS-10.13.6-380Gb-SIPoff_runner' + template_path: 'templates/macOS-10.13.json' + target_datastore: 'ds-image' + stable: false diff --git a/images.CI/macos/azure-pipelines/macos1014.yml b/images.CI/macos/azure-pipelines/macos1014.yml new file mode 100644 index 000000000..8149c55c2 --- /dev/null +++ b/images.CI/macos/azure-pipelines/macos1014.yml @@ -0,0 +1,23 @@ +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - main + always: true + +trigger: none +pr: + autoCancel: true + branches: + include: + - main + +jobs: +- template: image-generation.yml + parameters: + image_label: 'macOS Mojave' + base_image_name: 'clean-macOS-10.14-380Gb_runner' + template_path: 'templates/macOS-10.14.json' + target_datastore: 'ds-image' + stable: false diff --git a/images.CI/macos/azure-pipelines/macos1015.yml b/images.CI/macos/azure-pipelines/macos1015.yml new file mode 100644 index 000000000..7fc37016a --- /dev/null +++ b/images.CI/macos/azure-pipelines/macos1015.yml @@ -0,0 +1,23 @@ +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - main + always: true + +trigger: none +pr: + autoCancel: true + branches: + include: + - main + +jobs: +- template: image-generation.yml + parameters: + image_label: 'macOS Catalina' + base_image_name: 'clean-macOS-10.15-380Gb-runner' + template_path: 'templates/macOS-10.15.json' + target_datastore: 'ds-image' + stable: false From 77330e4553eff9fc7971341ed8a2114525431a22 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Fri, 11 Sep 2020 18:02:49 +0300 Subject: [PATCH 02/31] fix path --- images.CI/macos/azure-pipelines/image-generation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index b4af41203..40cf4d43a 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -25,7 +25,7 @@ jobs: - task: DeleteFiles@1 displayName: Clean up self-hosted machine inputs: - SourceFolder: 'image-generation/packer/provision/log/' + SourceFolder: 'images/macos/provision/log/' RemoveSourceFolder: true - bash: | @@ -50,7 +50,7 @@ jobs: -var="xcode_install_password=$(xcode-installation-password)" \ -color=false \ ${{ parameters.template_path }} - workingDirectory: 'image-generation/packer' + workingDirectory: 'images/macos' displayName: Build image - bash: | From b89beb150d6a62d6a37997aaedb78a6faf2f27e3 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 01:57:13 +0300 Subject: [PATCH 03/31] Update structure --- .../{linux-and-win => }/download-repo.ps1 | 0 .../azure-pipelines/image-generation.yml | 6 +- .../azure-pipelines/image-generation.yml | 54 ++++++++++-------- images.CI/macos/azure-pipelines/macos1013.yml | 3 +- images.CI/macos/azure-pipelines/macos1014.yml | 3 +- images.CI/macos/azure-pipelines/macos1015.yml | 3 +- images.CI/macos/build-image.ps1 | 55 +++++++++++++++++++ images.CI/macos/destroy_vm.sh | 18 ++++++ 8 files changed, 110 insertions(+), 32 deletions(-) rename images.CI/{linux-and-win => }/download-repo.ps1 (100%) create mode 100644 images.CI/macos/build-image.ps1 create mode 100644 images.CI/macos/destroy_vm.sh diff --git a/images.CI/linux-and-win/download-repo.ps1 b/images.CI/download-repo.ps1 similarity index 100% rename from images.CI/linux-and-win/download-repo.ps1 rename to images.CI/download-repo.ps1 diff --git a/images.CI/linux-and-win/azure-pipelines/image-generation.yml b/images.CI/linux-and-win/azure-pipelines/image-generation.yml index 30499ce3f..38aabd142 100644 --- a/images.CI/linux-and-win/azure-pipelines/image-generation.yml +++ b/images.CI/linux-and-win/azure-pipelines/image-generation.yml @@ -26,7 +26,7 @@ jobs: displayName: 'Build VM' inputs: targetType: filePath - filePath: ./images.CI/build-image.ps1 + filePath: ./images.CI/linux-and-win/build-image.ps1 arguments: -ResourcesNamePrefix $(Build.BuildId) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` @@ -45,7 +45,7 @@ jobs: displayName: 'Create release for VM deployment' inputs: targetType: filePath - filePath: ./images.CI/create-release.ps1 + filePath: ./images.CI/linux-and-win/create-release.ps1 arguments: -BuildId $(Build.BuildId) ` -Organization $(RELEASE_TARGET_ORGANIZATION) ` -DefinitionId $(RELEASE_TARGET_DEFINITION_ID) ` @@ -66,7 +66,7 @@ jobs: condition: always() inputs: targetType: filePath - filePath: ./images.CI/cleanup.ps1 + filePath: ./images.CI/linux-and-win/cleanup.ps1 arguments: -ResourcesNamePrefix $(Build.BuildId) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 40cf4d43a..df1c056e2 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -28,30 +28,30 @@ jobs: SourceFolder: 'images/macos/provision/log/' RemoveSourceFolder: true - - bash: | - export PACKER_LOG=0 - packer build -on-error=abort \ - -var="vcenter_server=$(vcenter_server_v2)" \ - -var="vcenter_username=$(vcenter_username_v2)" \ - -var="vcenter_password=$(vcenter_password_v2)" \ - -var="vcenter_datacenter=$(vcenter_datacenter_v2)" \ - -var="cluster_or_esxi_host=$(esxi_cluster_v2)" \ - -var="esxi_datastore=${{ parameters.target_datastore }}" \ - -var="output_folder=mms-output" \ - -var="vm_username=$(vm_username)" \ - -var="vm_password=$(vm_password)" \ - -var="build_id=$(Build.BuildNumber)" \ - -var="baseimage_name=${{ parameters.base_image_name }}" \ - -var="azure_storage_sas=$(AZURE_STORAGE_SAS)" \ - -var="azure_storage_account=$(AZURE_STORAGE_ACCOUNT)" \ - -var="build_assets_url=$(appcenter_build_assets_url)" \ - -var="github_feed_token=$(GITHUB_FEED_TOKEN)" \ - -var="xcode_install_user=$(xcode-installation-user)" \ - -var="xcode_install_password=$(xcode-installation-password)" \ - -color=false \ - ${{ parameters.template_path }} + - task: PowerShell@2 + displayName: 'Build VM' + inputs: + targetType: filePath + filePath: ./images.CI/macos/build-image.ps1 + arguments: -Image "${{ parameters.image_name }}" ` + -VcenterServer "$(vcenter_server_v2)" ` + -VcenterUsername "$(vcenter_username_v2)" ` + -VcenterPassword "$(vcenter_password_v2)" ` + -VcenterDatacenter "$(vcenter_datacenter_v2)" ` + -EsxiCluster "$(esxi_cluster_v2)" ` + -TargetDatastore "${{ parameters.target_datastore }}" ` + -OutputFolder "mms-output" ` + -VmUsername "$(vm_username)" ` + -VmPassword "$(vm_password)" ` + -BuildId "$(Build.BuildNumber)" ` + -BaseImageName "${{ parameters.base_image_name }}" ` + -AzureStorageSas "$(AZURE_STORAGE_SAS)" ` + -AzureStorageAccount "$(AZURE_STORAGE_ACCOUNT)" ` + -BuildAssetsUrl "$(appcenter_build_assets_url)" ` + -GithubFeedToken "$(GITHUB_FEED_TOKEN)" ` + -XcodeInstallationUser "$(xcode-installation-user)" ` + -XcodeInstallationPassword "$(xcode-installation-password)" workingDirectory: 'images/macos' - displayName: Build image - bash: | echo "Copy image output files" @@ -82,3 +82,11 @@ jobs: failTaskOnFailedTests: true displayName: Publish test results condition: always() + + - task: Bash@3 + inputs: + targetType: filePath + filePath: ./images.CI/macos/destroy_vm.sh + arguments: '"$(vcenter_server_v2)" "$(vcenter_username_v2)" "$(vcenter_password_v2)" "$(vcenter_datacenter_v2)/vm/mms-output/$(Build.BuildNumber)"' + condition: eq(variables['Agent.JobStatus'], 'Canceled') + displayName: Destroy VM (if build canceled only) \ No newline at end of file diff --git a/images.CI/macos/azure-pipelines/macos1013.yml b/images.CI/macos/azure-pipelines/macos1013.yml index 04a76346d..2dfeffdeb 100644 --- a/images.CI/macos/azure-pipelines/macos1013.yml +++ b/images.CI/macos/azure-pipelines/macos1013.yml @@ -18,6 +18,5 @@ jobs: parameters: image_label: 'macOS High Sierra' base_image_name: 'clean-macOS-10.13.6-380Gb-SIPoff_runner' - template_path: 'templates/macOS-10.13.json' + image_name: 'macOS-10.13' target_datastore: 'ds-image' - stable: false diff --git a/images.CI/macos/azure-pipelines/macos1014.yml b/images.CI/macos/azure-pipelines/macos1014.yml index 8149c55c2..521f7abf9 100644 --- a/images.CI/macos/azure-pipelines/macos1014.yml +++ b/images.CI/macos/azure-pipelines/macos1014.yml @@ -18,6 +18,5 @@ jobs: parameters: image_label: 'macOS Mojave' base_image_name: 'clean-macOS-10.14-380Gb_runner' - template_path: 'templates/macOS-10.14.json' + image_name: 'macOS-10.14' target_datastore: 'ds-image' - stable: false diff --git a/images.CI/macos/azure-pipelines/macos1015.yml b/images.CI/macos/azure-pipelines/macos1015.yml index 7fc37016a..eea21f4ea 100644 --- a/images.CI/macos/azure-pipelines/macos1015.yml +++ b/images.CI/macos/azure-pipelines/macos1015.yml @@ -18,6 +18,5 @@ jobs: parameters: image_label: 'macOS Catalina' base_image_name: 'clean-macOS-10.15-380Gb-runner' - template_path: 'templates/macOS-10.15.json' + image_name: 'macOS-10.15' target_datastore: 'ds-image' - stable: false diff --git a/images.CI/macos/build-image.ps1 b/images.CI/macos/build-image.ps1 new file mode 100644 index 000000000..eae76eebe --- /dev/null +++ b/images.CI/macos/build-image.ps1 @@ -0,0 +1,55 @@ +param( + [String] [Parameter (Mandatory=$true)] $Image, + [String] [Parameter (Mandatory=$true)] $VcenterServer, + [String] [Parameter (Mandatory=$true)] $VcenterUsername, + [String] [Parameter (Mandatory=$true)] $VcenterPassword, + [String] [Parameter (Mandatory=$true)] $VcenterDatacenter, + [String] [Parameter (Mandatory=$true)] $EsxiCluster, + [String] [Parameter (Mandatory=$true)] $TargetDatastore, + [String] [Parameter (Mandatory=$true)] $OutputFolder, + [String] [Parameter (Mandatory=$true)] $VmUsername, + [String] [Parameter (Mandatory=$true)] $VmPassword, + [String] [Parameter (Mandatory=$true)] $BuildId, + [String] [Parameter (Mandatory=$true)] $BaseImageName, + [String] [Parameter (Mandatory=$true)] $AzureStorageSas, + [String] [Parameter (Mandatory=$true)] $AzureStorageAccount, + [String] [Parameter (Mandatory=$true)] $BuildAssetsUrl, + [String] [Parameter (Mandatory=$true)] $GithubFeedToken, + [String] [Parameter (Mandatory=$true)] $XcodeInstallationUser, + [String] [Parameter (Mandatory=$true)] $XcodeInstallationPassword +) + +$TemplatePath = (Get-ChildItem -Path "images" -Include "$Image.json" -Recurse -Depth 2).FullName +if (-not $TemplatePath) +{ + Write-Error "'-Image' parameter is not valid. You have to specify correct image type." + exit 1 +} + +$env:PACKER_LOG = 0 + +packer validate -syntax-only $TemplatePath + +Write-Host "Show Packer Version" +packer --version + +Write-Host "Build $Image VM" +packer build -on-error=abort -var="vcenter_server=$VcenterServer" ` + -var="vcenter_username=$VcenterUsername" ` + -var="vcenter_password=$VcenterPassword" ` + -var="vcenter_datacenter=$VcenterDatacenter" ` + -var="cluster_or_esxi_host=$EsxiCluster" ` + -var="esxi_datastore=$TargetDatastore" ` + -var="output_folder=$OutputFolder" ` + -var="vm_username=$VmUsername" ` + -var="vm_password=$VmPassword" ` + -var="build_id=$BuildId" ` + -var="baseimage_name=$BaseImageName" ` + -var="azure_storage_sas=$AzureStorageSas" ` + -var="azure_storage_account=$AzureStorageAccount" ` + -var="build_assets_url=$BuildAssetsUrl" ` + -var="github_feed_token=$GithubFeedToken" ` + -var="xcode_install_user=$XcodeInstallationUser" ` + -var="xcode_install_password=$XcodeInstallationPassword" ` + -color=false ` + $TemplatePath \ No newline at end of file diff --git a/images.CI/macos/destroy_vm.sh b/images.CI/macos/destroy_vm.sh new file mode 100644 index 000000000..cd7863cda --- /dev/null +++ b/images.CI/macos/destroy_vm.sh @@ -0,0 +1,18 @@ +VCENTER_SERVER=$1 +VCENTER_USERNAME=$2 +VCENTER_PASSWORD=$3 +VM_IPATH=$4 + +URLENCODE="image-generation/bootstrap/support/urlencode.sh" + +govc vm.power -k=true \ +-off=true \ +-u "$VCENTER_USERNAME:`$URLENCODE "$VCENTER_PASSWORD"`@$VCENTER_SERVER" \ +-vm.ipath="$VM_IPATH" + +#Time to VM's shutdown +sleep 30 + +govc vm.destroy -k=true \ +-u "$VCENTER_USERNAME:`$URLENCODE "$VCENTER_PASSWORD"`@$VCENTER_SERVER" \ +-vm.ipath="$VM_IPATH" \ No newline at end of file From 7f3dce97050097d6ea1a149a3ceba1d2074e3c2e Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 02:09:16 +0300 Subject: [PATCH 04/31] Fix build VM step --- .../azure-pipelines/image-generation.yml | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index df1c056e2..eb95bf621 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -28,29 +28,28 @@ jobs: SourceFolder: 'images/macos/provision/log/' RemoveSourceFolder: true - - task: PowerShell@2 + - pwsh: | + ./images.CI/macos/build-image.ps1 -Image "${{ parameters.image_name }}" ` + -VcenterServer "$(vcenter_server_v2)" ` + -VcenterUsername "$(vcenter_username_v2)" ` + -VcenterPassword "$(vcenter_password_v2)" ` + -VcenterDatacenter "$(vcenter_datacenter_v2)" ` + -EsxiCluster "$(esxi_cluster_v2)" ` + -TargetDatastore "${{ parameters.target_datastore }}" ` + -OutputFolder "mms-output" ` + -VmUsername "$(vm_username)" ` + -VmPassword "$(vm_password)" ` + -BuildId "$(Build.BuildNumber)" ` + -BaseImageName "${{ parameters.base_image_name }}" ` + -AzureStorageSas "$(AZURE_STORAGE_SAS)" ` + -AzureStorageAccount "$(AZURE_STORAGE_ACCOUNT)" ` + -BuildAssetsUrl "$(appcenter_build_assets_url)" ` + -GithubFeedToken "$(GITHUB_FEED_TOKEN)" ` + -XcodeInstallationUser "$(xcode-installation-user)" ` + -XcodeInstallationPassword "$(xcode-installation-password)" displayName: 'Build VM' - inputs: - targetType: filePath - filePath: ./images.CI/macos/build-image.ps1 - arguments: -Image "${{ parameters.image_name }}" ` - -VcenterServer "$(vcenter_server_v2)" ` - -VcenterUsername "$(vcenter_username_v2)" ` - -VcenterPassword "$(vcenter_password_v2)" ` - -VcenterDatacenter "$(vcenter_datacenter_v2)" ` - -EsxiCluster "$(esxi_cluster_v2)" ` - -TargetDatastore "${{ parameters.target_datastore }}" ` - -OutputFolder "mms-output" ` - -VmUsername "$(vm_username)" ` - -VmPassword "$(vm_password)" ` - -BuildId "$(Build.BuildNumber)" ` - -BaseImageName "${{ parameters.base_image_name }}" ` - -AzureStorageSas "$(AZURE_STORAGE_SAS)" ` - -AzureStorageAccount "$(AZURE_STORAGE_ACCOUNT)" ` - -BuildAssetsUrl "$(appcenter_build_assets_url)" ` - -GithubFeedToken "$(GITHUB_FEED_TOKEN)" ` - -XcodeInstallationUser "$(xcode-installation-user)" ` - -XcodeInstallationPassword "$(xcode-installation-password)" + env: + PACKER_LOG: 0 workingDirectory: 'images/macos' - bash: | From e918791a0a1529c22c5596626b2aa69bbfff4542 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 02:13:24 +0300 Subject: [PATCH 05/31] fix path to the script --- .../azure-pipelines/image-generation.yml | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index eb95bf621..4ee75821e 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -29,24 +29,24 @@ jobs: RemoveSourceFolder: true - pwsh: | - ./images.CI/macos/build-image.ps1 -Image "${{ parameters.image_name }}" ` - -VcenterServer "$(vcenter_server_v2)" ` - -VcenterUsername "$(vcenter_username_v2)" ` - -VcenterPassword "$(vcenter_password_v2)" ` - -VcenterDatacenter "$(vcenter_datacenter_v2)" ` - -EsxiCluster "$(esxi_cluster_v2)" ` - -TargetDatastore "${{ parameters.target_datastore }}" ` - -OutputFolder "mms-output" ` - -VmUsername "$(vm_username)" ` - -VmPassword "$(vm_password)" ` - -BuildId "$(Build.BuildNumber)" ` - -BaseImageName "${{ parameters.base_image_name }}" ` - -AzureStorageSas "$(AZURE_STORAGE_SAS)" ` - -AzureStorageAccount "$(AZURE_STORAGE_ACCOUNT)" ` - -BuildAssetsUrl "$(appcenter_build_assets_url)" ` - -GithubFeedToken "$(GITHUB_FEED_TOKEN)" ` - -XcodeInstallationUser "$(xcode-installation-user)" ` - -XcodeInstallationPassword "$(xcode-installation-password)" + $(build.sourcesDirectory)/images.CI/macos/build-image.ps1 -Image "${{ parameters.image_name }}" ` + -VcenterServer "$(vcenter_server_v2)" ` + -VcenterUsername "$(vcenter_username_v2)" ` + -VcenterPassword "$(vcenter_password_v2)" ` + -VcenterDatacenter "$(vcenter_datacenter_v2)" ` + -EsxiCluster "$(esxi_cluster_v2)" ` + -TargetDatastore "${{ parameters.target_datastore }}" ` + -OutputFolder "mms-output" ` + -VmUsername "$(vm_username)" ` + -VmPassword "$(vm_password)" ` + -BuildId "$(Build.BuildNumber)" ` + -BaseImageName "${{ parameters.base_image_name }}" ` + -AzureStorageSas "$(AZURE_STORAGE_SAS)" ` + -AzureStorageAccount "$(AZURE_STORAGE_ACCOUNT)" ` + -BuildAssetsUrl "$(appcenter_build_assets_url)" ` + -GithubFeedToken "$(GITHUB_FEED_TOKEN)" ` + -XcodeInstallationUser "$(xcode-installation-user)" ` + -XcodeInstallationPassword "$(xcode-installation-password)" displayName: 'Build VM' env: PACKER_LOG: 0 From 3f3e73819a4a6ccacb26b449cea739837036e4a4 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 02:15:55 +0300 Subject: [PATCH 06/31] fix build-image.ps1 script --- images.CI/macos/build-image.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images.CI/macos/build-image.ps1 b/images.CI/macos/build-image.ps1 index eae76eebe..1e462f9d4 100644 --- a/images.CI/macos/build-image.ps1 +++ b/images.CI/macos/build-image.ps1 @@ -19,7 +19,7 @@ param( [String] [Parameter (Mandatory=$true)] $XcodeInstallationPassword ) -$TemplatePath = (Get-ChildItem -Path "images" -Include "$Image.json" -Recurse -Depth 2).FullName +$TemplatePath = (Get-ChildItem -Path "templates" -Include "$Image.json" -Recurse -Depth 2).FullName if (-not $TemplatePath) { Write-Error "'-Image' parameter is not valid. You have to specify correct image type." From ab5c601f875bd87d74e5d701d97ef20f83d41584 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 02:23:17 +0300 Subject: [PATCH 07/31] Remove destroy vm script --- .../macos/azure-pipelines/image-generation.yml | 8 -------- images.CI/macos/destroy_vm.sh | 18 ------------------ 2 files changed, 26 deletions(-) delete mode 100644 images.CI/macos/destroy_vm.sh diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 4ee75821e..98aec59b8 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -81,11 +81,3 @@ jobs: failTaskOnFailedTests: true displayName: Publish test results condition: always() - - - task: Bash@3 - inputs: - targetType: filePath - filePath: ./images.CI/macos/destroy_vm.sh - arguments: '"$(vcenter_server_v2)" "$(vcenter_username_v2)" "$(vcenter_password_v2)" "$(vcenter_datacenter_v2)/vm/mms-output/$(Build.BuildNumber)"' - condition: eq(variables['Agent.JobStatus'], 'Canceled') - displayName: Destroy VM (if build canceled only) \ No newline at end of file diff --git a/images.CI/macos/destroy_vm.sh b/images.CI/macos/destroy_vm.sh deleted file mode 100644 index cd7863cda..000000000 --- a/images.CI/macos/destroy_vm.sh +++ /dev/null @@ -1,18 +0,0 @@ -VCENTER_SERVER=$1 -VCENTER_USERNAME=$2 -VCENTER_PASSWORD=$3 -VM_IPATH=$4 - -URLENCODE="image-generation/bootstrap/support/urlencode.sh" - -govc vm.power -k=true \ --off=true \ --u "$VCENTER_USERNAME:`$URLENCODE "$VCENTER_PASSWORD"`@$VCENTER_SERVER" \ --vm.ipath="$VM_IPATH" - -#Time to VM's shutdown -sleep 30 - -govc vm.destroy -k=true \ --u "$VCENTER_USERNAME:`$URLENCODE "$VCENTER_PASSWORD"`@$VCENTER_SERVER" \ --vm.ipath="$VM_IPATH" \ No newline at end of file From 1f73b64e5c254d912d3db1b4defc9d9086639270 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Mon, 14 Sep 2020 11:01:14 +0300 Subject: [PATCH 08/31] Improved documentation for Android --- .../SoftwareReport.Android.psm1 | 202 +++++++++++------- .../SoftwareReport.Generator.ps1 | 28 +-- 2 files changed, 129 insertions(+), 101 deletions(-) diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 b/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 index cc9fbb15d..7e799e8ba 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 @@ -1,3 +1,10 @@ +function Split-TableRowByColumns { + param( + [string] $Row + ) + return $Row.Split("|") | ForEach-Object { $_.trim() } +} + function Get-AndroidComponentLocation { param( [string] $ComponentName @@ -6,22 +13,12 @@ function Get-AndroidComponentLocation { return "Location $path" } -function Split-AndroidSDKOutputRow { +function Get-AndroidSDKRoot { param( - [string] $Row + [string] $ComponentName ) - return $Row.Split("|").Trim() -} - -function Create-AndroidTableObject { - param( - [string] $PackageName, - [string] $Description - ) - return [PSCustomObject] @{ - "Package Name" = $PackageName - "Description" = $Description - } + $path = Join-Path $env:ANDROID_HOME $ComponentName + return "Location $path" } function Get-AndroidSDKManagerPath { @@ -42,68 +39,123 @@ function Get-AndroidInstalledPackages { return $androidInstalledPackages } -function Build-AndroidSDKToolsTable { - param ( - [Parameter(Mandatory)] - [object] $packageInfo - ) - return $packageInfo | ForEach-Object { - $packageInfoParts = Split-AndroidSDKOutputRow $_ - $packageName = $packageInfoParts[0] - $packageDescription = $packageInfoParts[2] + ", Revision " + $packageInfoParts[1] - Create-AndroidTableObject -PackageName $packageName -Description $packageDescription - } -} - -function Build-AndroidSDKPlatformTable { - param ( - [Parameter(Mandatory)] - [object] $packageInfo - ) - - return $packageInfo | ForEach-Object { - $packageInfoParts = Split-AndroidSDKOutputRow $_ - $packageName = $packageInfoParts[0].split(";")[1] - $packageDescription = $packageInfoParts[2] + ", Revision " + $packageInfoParts[1] - return Create-AndroidTableObject -PackageName $packageName -Description $packageDescription - } -} - -function Build-AndroidSDKBuildToolsTable { - param ( - [Parameter(Mandatory)] - [object] $packageInfo - ) - - return $packageInfo | ForEach-Object { - $packageInfoParts = Split-AndroidSDKOutputRow $_ - $packageName = $packageInfoParts[0].replace(";", "-") - $packageDescription = "Android SDK Build-Tools, Revision " + $packageInfoParts[1] - return Create-AndroidTableObject -PackageName $packageName -Description $packageDescription - } -} - -function Build-AndroidExtraPackagesTable { - param ( - [Parameter(Mandatory)][AllowEmptyString()] - [string[]] $installedPackages - ) - - $extraPackages = @( - "Android Support Repository", - "Google Play services", - "Google Repository", - "ndk-bundle" - ) - - return $extraPackages | ForEach-Object { - $packageId = $_ - $packageInfo = $installedPackages | Where-Object { $_ -Like "*${packageId}*" } | Select-Object -First 1 - $packageInfoParts = Split-AndroidSDKOutputRow $packageInfo - return [PSCustomObject] @{ - "Package Name" = $packageInfoParts[2] - "Version" = $packageInfoParts[1] +function Build-AndroidTable { + $packageInfo = Get-AndroidInstalledPackages + return @( + @{ + "Package" = "Android SDK Tools" + "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Android SDK Tools" + }, + @{ + "Package" = "Android SDK Platforms" + "Version" = Get-AndroidPlatformVersions -PackageInfo $packageInfo + "Location" = Get-AndroidComponentLocation -ComponentName "platforms" + }, + @{ + "Package" = "Android SDK Build-tools" + "Version" = Get-AndroidBuildToolVersions -PackageInfo $packageInfo + "Location" = Get-AndroidComponentLocation -ComponentName "build-tools" + }, + @{ + "Package" = "Android SDK Platform-Tools" + "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Android SDK Platform-Tools" + }, + @{ + "Package" = "Google APIs" + "Version" = Get-AndroidGoogleAPIsVersions -PackageInfo $packageInfo + }, + @{ + "Package" = "Android Support Repository" + "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Android Support Repository" + }, + @{ + "Package" = "Google Play services" + "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Google Play services" + }, + @{ + "Package" = "Google Repository" + "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Google Repository" + }, + @{ + "Package" = "SDK Patch Applier v4" + "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "SDK Patch Applier v4" + }, + @{ + "Package" = "CMake" + "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "cmake" + }, + @{ + "Package" = "NDK" + "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "ndk-bundle" + } + ) | Where-Object { $_.Version } | ForEach-Object { + [PSCustomObject] @{ + "Package Name" = $_.Package + "Version" = $_.Version + "Location" = $_.Location } } +} + +function Get-AndroidPackageVersions { + param ( + [Parameter(Mandatory)] + [object] $PackageInfo, + [Parameter(Mandatory)] + [object] $MatchedString + ) + + $versions = $packageInfo | Where-Object { $_ -Match $MatchedString } | ForEach-Object { + $packageInfoParts = Split-TableRowByColumns $_ + return $packageInfoParts[1] + } + return ($versions -Join "
") +} + +function Get-AndroidPlatformVersions { + param ( + [Parameter(Mandatory)] + [object] $PackageInfo + ) + + $versions = $packageInfo | Where-Object { $_ -Match "Android SDK Platform " } | ForEach-Object { + $packageInfoParts = Split-TableRowByColumns $_ + $revision = $packageInfoParts[1] + $version = $packageInfoParts[0].split(";")[1] + return "$version, (rev $revision)" + } + [array]::Reverse($versions) + return ($versions -Join "
") +} + +function Get-AndroidBuildToolVersions { + param ( + [Parameter(Mandatory)] + [object] $PackageInfo + ) + + $versions = $packageInfo | Where-Object { $_ -Match "Android SDK Build-Tools" } | ForEach-Object { + $packageInfoParts = Split-TableRowByColumns $_ + return $packageInfoParts[1] + } + $groupVersions = @() + $versions | ForEach-Object { + $majorVersion = $_.Split(".")[0] + $groupVersions += $versions | Where-Object { $_.StartsWith($majorVersion) } | Join-String -Separator " " + } + return ($groupVersions | Sort-Object -Descending -Unique | Join-String -Separator "
") +} + +function Get-AndroidGoogleAPIsVersions { + param ( + [Parameter(Mandatory)] + [object] $PackageInfo + ) + + $versions = $packageInfo | Where-Object { $_ -Match "Google APIs" } | ForEach-Object { + $packageInfoParts = Split-TableRowByColumns $_ + return $packageInfoParts[0].split(";")[1] + } + return ($versions -Join "
") } \ No newline at end of file diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 index 5ff886c77..c93e5b8a2 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 @@ -227,32 +227,8 @@ $markdown += Get-PowerShellModules | New-MDTable $markdown += New-MDNewLine # Android section -$androidInstalledPackages = Get-AndroidInstalledPackages - -$markdown += New-MDHeader "Android SDK Tools" -Level 3 -$androidSDKTools = $androidInstalledPackages | Where-Object { $_ -Match "Android SDK Tools" -or $_ -Match "Android SDK Platform-Tools" } -$markdown += Build-AndroidSDKToolsTable $androidSDKTools | New-MDTable +$markdown += New-MDHeader "Android" -Level 3 +$markdown += Build-AndroidTable | New-MDTable $markdown += New-MDNewLine -$markdown += New-MDHeader "Android SDK Platforms" -Level 3 -$androidSDKPlatforms = $androidInstalledPackages | Where-Object { $_ -Match "Android SDK Platform " } -$markdown += New-MDInlineCode -Text (Get-AndroidComponentLocation -ComponentName "platforms") -$markdown += New-MDNewLine -$markdown += Build-AndroidSDKPlatformTable $androidSDKPlatforms | New-MDTable -$markdown += New-MDNewLine - -$markdown += New-MDHeader "Android SDK Build-Tools" -Level 3 -$androidSDKBuildTools = $androidInstalledPackages | Where-Object { $_ -Match "Android SDK Build-Tools" } -$markdown += New-MDInlineCode -Text (Get-AndroidComponentLocation -ComponentName "build-tools") -$markdown += New-MDNewLine -$markdown += Build-AndroidSDKBuildtoolsTable $androidSDKBuildTools | New-MDTable -$markdown += New-MDNewLine - -$markdown += New-MDHeader "Android Extra Packages" -Level 3 -$markdown += Build-AndroidExtraPackagesTable $androidInstalledPackages | New-MDTable -$markdown += New-MDNewLine - -$markdown += New-MDHeader "Cached Docker images" -Level 3 -$markdown += New-MDList -Style Unordered -Lines @(Get-CachedDockerImages) - $markdown | Out-File -FilePath "C:\InstalledSoftware.md" \ No newline at end of file From 7e0e8fcab61ca9c482d8d605c3fc1353b939c1a5 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 15:44:26 +0300 Subject: [PATCH 09/31] Fix structure --- .../azure-pipelines/image-generation.yml | 41 +++++++------- images.CI/macos/azure-pipelines/macos1013.yml | 2 +- images.CI/macos/azure-pipelines/macos1014.yml | 2 +- images.CI/macos/azure-pipelines/macos1015.yml | 2 +- images.CI/macos/azure-pipelines/macos110.yml | 22 ++++++++ images.CI/macos/build-image.ps1 | 55 ------------------- 6 files changed, 45 insertions(+), 79 deletions(-) create mode 100644 images.CI/macos/azure-pipelines/macos110.yml delete mode 100644 images.CI/macos/build-image.ps1 diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 98aec59b8..bd39df574 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -28,25 +28,24 @@ jobs: SourceFolder: 'images/macos/provision/log/' RemoveSourceFolder: true - - pwsh: | - $(build.sourcesDirectory)/images.CI/macos/build-image.ps1 -Image "${{ parameters.image_name }}" ` - -VcenterServer "$(vcenter_server_v2)" ` - -VcenterUsername "$(vcenter_username_v2)" ` - -VcenterPassword "$(vcenter_password_v2)" ` - -VcenterDatacenter "$(vcenter_datacenter_v2)" ` - -EsxiCluster "$(esxi_cluster_v2)" ` - -TargetDatastore "${{ parameters.target_datastore }}" ` - -OutputFolder "mms-output" ` - -VmUsername "$(vm_username)" ` - -VmPassword "$(vm_password)" ` - -BuildId "$(Build.BuildNumber)" ` - -BaseImageName "${{ parameters.base_image_name }}" ` - -AzureStorageSas "$(AZURE_STORAGE_SAS)" ` - -AzureStorageAccount "$(AZURE_STORAGE_ACCOUNT)" ` - -BuildAssetsUrl "$(appcenter_build_assets_url)" ` - -GithubFeedToken "$(GITHUB_FEED_TOKEN)" ` - -XcodeInstallationUser "$(xcode-installation-user)" ` - -XcodeInstallationPassword "$(xcode-installation-password)" + - bash: | + packer build -on-error=abort \ + -var="vcenter_server=$(vcenter_server_v2)" \ + -var="vcenter_username=$(vcenter_username_v2)" \ + -var="vcenter_password=$(vcenter_password_v2)" \ + -var="vcenter_datacenter=$(vcenter_datacenter_v2)" \ + -var="cluster_or_esxi_host=$(esxi_cluster_v2)" \ + -var="esxi_datastore=${{ parameters.target_datastore }}" \ + -var="output_folder=mms-output" \ + -var="vm_username=$(vm_username)" \ + -var="vm_password=$(vm_password)" \ + -var="build_id=$(Build.BuildNumber)" \ + -var="baseimage_name=${{ parameters.base_image_name }}" \ + -var="github_feed_token=$(GITHUB_FEED_TOKEN)" \ + -var="xcode_install_user=$(xcode-installation-user)" \ + -var="xcode_install_password=$(xcode-installation-password)" \ + -color=false \ + ${{ parameters.template_path }} displayName: 'Build VM' env: PACKER_LOG: 0 @@ -54,10 +53,10 @@ jobs: - bash: | echo "Copy image output files" - cp -R "image-generation/image-output/software-report/." "$(Build.ArtifactStagingDirectory)/" + cp -R "images/image-output/software-report/." "$(Build.ArtifactStagingDirectory)/" echo "Copy test results" - cp -R "image-generation/image-output/tests/." "$(Common.TestResultsDirectory)/" + cp -R "images/image-output/tests/." "$(Common.TestResultsDirectory)/" ls $(Common.TestResultsDirectory) echo "Put VM name to 'VM_Done_Name' file" diff --git a/images.CI/macos/azure-pipelines/macos1013.yml b/images.CI/macos/azure-pipelines/macos1013.yml index 2dfeffdeb..d214bccf6 100644 --- a/images.CI/macos/azure-pipelines/macos1013.yml +++ b/images.CI/macos/azure-pipelines/macos1013.yml @@ -18,5 +18,5 @@ jobs: parameters: image_label: 'macOS High Sierra' base_image_name: 'clean-macOS-10.13.6-380Gb-SIPoff_runner' - image_name: 'macOS-10.13' + template_path: 'templates/macOS-10.13.json' target_datastore: 'ds-image' diff --git a/images.CI/macos/azure-pipelines/macos1014.yml b/images.CI/macos/azure-pipelines/macos1014.yml index 521f7abf9..9ee818eb7 100644 --- a/images.CI/macos/azure-pipelines/macos1014.yml +++ b/images.CI/macos/azure-pipelines/macos1014.yml @@ -18,5 +18,5 @@ jobs: parameters: image_label: 'macOS Mojave' base_image_name: 'clean-macOS-10.14-380Gb_runner' - image_name: 'macOS-10.14' + template_path: 'templates/macOS-10.14.json' target_datastore: 'ds-image' diff --git a/images.CI/macos/azure-pipelines/macos1015.yml b/images.CI/macos/azure-pipelines/macos1015.yml index eea21f4ea..967f741b9 100644 --- a/images.CI/macos/azure-pipelines/macos1015.yml +++ b/images.CI/macos/azure-pipelines/macos1015.yml @@ -18,5 +18,5 @@ jobs: parameters: image_label: 'macOS Catalina' base_image_name: 'clean-macOS-10.15-380Gb-runner' - image_name: 'macOS-10.15' + template_path: 'templates/macOS-10.15.json' target_datastore: 'ds-image' diff --git a/images.CI/macos/azure-pipelines/macos110.yml b/images.CI/macos/azure-pipelines/macos110.yml new file mode 100644 index 000000000..7a7864270 --- /dev/null +++ b/images.CI/macos/azure-pipelines/macos110.yml @@ -0,0 +1,22 @@ +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - main + always: true + +trigger: none +pr: + autoCancel: true + branches: + include: + - main + +jobs: +- template: image-generation-job.yml + parameters: + image_label: 'macOS Big Sur' + base_image_name: 'clean-macOS-11.0-380Gb-runner' + template_path: 'templates/macOS-11.0.json' + target_datastore: 'ds-image' diff --git a/images.CI/macos/build-image.ps1 b/images.CI/macos/build-image.ps1 deleted file mode 100644 index 1e462f9d4..000000000 --- a/images.CI/macos/build-image.ps1 +++ /dev/null @@ -1,55 +0,0 @@ -param( - [String] [Parameter (Mandatory=$true)] $Image, - [String] [Parameter (Mandatory=$true)] $VcenterServer, - [String] [Parameter (Mandatory=$true)] $VcenterUsername, - [String] [Parameter (Mandatory=$true)] $VcenterPassword, - [String] [Parameter (Mandatory=$true)] $VcenterDatacenter, - [String] [Parameter (Mandatory=$true)] $EsxiCluster, - [String] [Parameter (Mandatory=$true)] $TargetDatastore, - [String] [Parameter (Mandatory=$true)] $OutputFolder, - [String] [Parameter (Mandatory=$true)] $VmUsername, - [String] [Parameter (Mandatory=$true)] $VmPassword, - [String] [Parameter (Mandatory=$true)] $BuildId, - [String] [Parameter (Mandatory=$true)] $BaseImageName, - [String] [Parameter (Mandatory=$true)] $AzureStorageSas, - [String] [Parameter (Mandatory=$true)] $AzureStorageAccount, - [String] [Parameter (Mandatory=$true)] $BuildAssetsUrl, - [String] [Parameter (Mandatory=$true)] $GithubFeedToken, - [String] [Parameter (Mandatory=$true)] $XcodeInstallationUser, - [String] [Parameter (Mandatory=$true)] $XcodeInstallationPassword -) - -$TemplatePath = (Get-ChildItem -Path "templates" -Include "$Image.json" -Recurse -Depth 2).FullName -if (-not $TemplatePath) -{ - Write-Error "'-Image' parameter is not valid. You have to specify correct image type." - exit 1 -} - -$env:PACKER_LOG = 0 - -packer validate -syntax-only $TemplatePath - -Write-Host "Show Packer Version" -packer --version - -Write-Host "Build $Image VM" -packer build -on-error=abort -var="vcenter_server=$VcenterServer" ` - -var="vcenter_username=$VcenterUsername" ` - -var="vcenter_password=$VcenterPassword" ` - -var="vcenter_datacenter=$VcenterDatacenter" ` - -var="cluster_or_esxi_host=$EsxiCluster" ` - -var="esxi_datastore=$TargetDatastore" ` - -var="output_folder=$OutputFolder" ` - -var="vm_username=$VmUsername" ` - -var="vm_password=$VmPassword" ` - -var="build_id=$BuildId" ` - -var="baseimage_name=$BaseImageName" ` - -var="azure_storage_sas=$AzureStorageSas" ` - -var="azure_storage_account=$AzureStorageAccount" ` - -var="build_assets_url=$BuildAssetsUrl" ` - -var="github_feed_token=$GithubFeedToken" ` - -var="xcode_install_user=$XcodeInstallationUser" ` - -var="xcode_install_password=$XcodeInstallationPassword" ` - -color=false ` - $TemplatePath \ No newline at end of file From eff7f1d78339cfcab4e79be3f5ccae63142ef996 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 16:46:44 +0300 Subject: [PATCH 10/31] Add name to ymls and add output_folder variable --- images.CI/macos/azure-pipelines/image-generation.yml | 2 +- images.CI/macos/azure-pipelines/macos1013.yml | 1 + images.CI/macos/azure-pipelines/macos1014.yml | 1 + images.CI/macos/azure-pipelines/macos1015.yml | 1 + images.CI/macos/azure-pipelines/macos110.yml | 3 ++- 5 files changed, 6 insertions(+), 2 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index bd39df574..345656951 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -36,7 +36,7 @@ jobs: -var="vcenter_datacenter=$(vcenter_datacenter_v2)" \ -var="cluster_or_esxi_host=$(esxi_cluster_v2)" \ -var="esxi_datastore=${{ parameters.target_datastore }}" \ - -var="output_folder=mms-output" \ + -var="output_folder=$(output_folder)" \ -var="vm_username=$(vm_username)" \ -var="vm_password=$(vm_password)" \ -var="build_id=$(Build.BuildNumber)" \ diff --git a/images.CI/macos/azure-pipelines/macos1013.yml b/images.CI/macos/azure-pipelines/macos1013.yml index d214bccf6..a7b9a4a19 100644 --- a/images.CI/macos/azure-pipelines/macos1013.yml +++ b/images.CI/macos/azure-pipelines/macos1013.yml @@ -1,3 +1,4 @@ +name: macOS-10.13_$(date:yyyyMMdd)$(rev:.r)_unstable schedules: - cron: "0 0 * * *" displayName: Daily diff --git a/images.CI/macos/azure-pipelines/macos1014.yml b/images.CI/macos/azure-pipelines/macos1014.yml index 9ee818eb7..21f7e48c1 100644 --- a/images.CI/macos/azure-pipelines/macos1014.yml +++ b/images.CI/macos/azure-pipelines/macos1014.yml @@ -1,3 +1,4 @@ +name: macOS-10.14_$(date:yyyyMMdd)$(rev:.r)_unstable schedules: - cron: "0 0 * * *" displayName: Daily diff --git a/images.CI/macos/azure-pipelines/macos1015.yml b/images.CI/macos/azure-pipelines/macos1015.yml index 967f741b9..f42980cc4 100644 --- a/images.CI/macos/azure-pipelines/macos1015.yml +++ b/images.CI/macos/azure-pipelines/macos1015.yml @@ -1,3 +1,4 @@ +name: macOS-10.15_$(date:yyyyMMdd)$(rev:.r)_unstable schedules: - cron: "0 0 * * *" displayName: Daily diff --git a/images.CI/macos/azure-pipelines/macos110.yml b/images.CI/macos/azure-pipelines/macos110.yml index 7a7864270..ada6dcfe1 100644 --- a/images.CI/macos/azure-pipelines/macos110.yml +++ b/images.CI/macos/azure-pipelines/macos110.yml @@ -1,3 +1,4 @@ +name: macOS-11.0_$(date:yyyyMMdd)$(rev:.r)_unstable schedules: - cron: "0 0 * * *" displayName: Daily @@ -14,7 +15,7 @@ pr: - main jobs: -- template: image-generation-job.yml +- template: image-generation.yml parameters: image_label: 'macOS Big Sur' base_image_name: 'clean-macOS-11.0-380Gb-runner' From 221a58cd32dfa66e8cf21e854d4605eb4d61f8d7 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 22:06:19 +0300 Subject: [PATCH 11/31] Remove sensitive data from logs --- .../macos/azure-pipelines/image-generation.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 345656951..8236a7b7c 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -28,7 +28,12 @@ jobs: SourceFolder: 'images/macos/provision/log/' RemoveSourceFolder: true - - bash: | + - pwsh: | + $SensitiveData = @( + 'IP address:', + 'Using ssh communicator to connect:' + ) + packer build -on-error=abort \ -var="vcenter_server=$(vcenter_server_v2)" \ -var="vcenter_username=$(vcenter_username_v2)" \ @@ -45,7 +50,13 @@ jobs: -var="xcode_install_user=$(xcode-installation-user)" \ -var="xcode_install_password=$(xcode-installation-password)" \ -color=false \ - ${{ parameters.template_path }} + ${{ parameters.template_path }} ` + | Where-Object { + #Filter sensitive data from Packer logs + $currentString = $_ + $sensitiveString = $SensitiveData | Where-Object { $currentString -match $_ } + $sensitiveString -eq $null + } displayName: 'Build VM' env: PACKER_LOG: 0 From d166ca23b751942b207429518c4c0930a51dc49f Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 14 Sep 2020 22:11:50 +0300 Subject: [PATCH 12/31] fix typo --- .../azure-pipelines/image-generation.yml | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 8236a7b7c..8eb3f6998 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -34,22 +34,22 @@ jobs: 'Using ssh communicator to connect:' ) - packer build -on-error=abort \ - -var="vcenter_server=$(vcenter_server_v2)" \ - -var="vcenter_username=$(vcenter_username_v2)" \ - -var="vcenter_password=$(vcenter_password_v2)" \ - -var="vcenter_datacenter=$(vcenter_datacenter_v2)" \ - -var="cluster_or_esxi_host=$(esxi_cluster_v2)" \ - -var="esxi_datastore=${{ parameters.target_datastore }}" \ - -var="output_folder=$(output_folder)" \ - -var="vm_username=$(vm_username)" \ - -var="vm_password=$(vm_password)" \ - -var="build_id=$(Build.BuildNumber)" \ - -var="baseimage_name=${{ parameters.base_image_name }}" \ - -var="github_feed_token=$(GITHUB_FEED_TOKEN)" \ - -var="xcode_install_user=$(xcode-installation-user)" \ - -var="xcode_install_password=$(xcode-installation-password)" \ - -color=false \ + packer build -on-error=abort ` + -var="vcenter_server=$(vcenter_server_v2)" ` + -var="vcenter_username=$(vcenter_username_v2)" ` + -var="vcenter_password=$(vcenter_password_v2)" ` + -var="vcenter_datacenter=$(vcenter_datacenter_v2)" ` + -var="cluster_or_esxi_host=$(esxi_cluster_v2)" ` + -var="esxi_datastore=${{ parameters.target_datastore }}" ` + -var="output_folder=$(output_folder)" ` + -var="vm_username=$(vm_username)" ` + -var="vm_password=$(vm_password)" ` + -var="build_id=$(Build.BuildNumber)" ` + -var="baseimage_name=${{ parameters.base_image_name }}" ` + -var="github_feed_token=$(GITHUB_FEED_TOKEN)" ` + -var="xcode_install_user=$(xcode-installation-user)" ` + -var="xcode_install_password=$(xcode-installation-password)" ` + -color=false ` ${{ parameters.template_path }} ` | Where-Object { #Filter sensitive data from Packer logs From 6b0d1e5a780ae4c4cb0d4ec6713b3964cd188577 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Tue, 15 Sep 2020 12:03:42 +0300 Subject: [PATCH 13/31] removed Location column --- .../SoftwareReport.Android.psm1 | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 b/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 index 7e799e8ba..d27ac6736 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 @@ -43,6 +43,10 @@ function Get-AndroidInstalledPackages { function Build-AndroidTable { $packageInfo = Get-AndroidInstalledPackages return @( + @{ + "Package" = "Android SDK Platform-Tools" + "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Android SDK Platform-Tools" + }, @{ "Package" = "Android SDK Tools" "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Android SDK Tools" @@ -50,25 +54,23 @@ function Build-AndroidTable { @{ "Package" = "Android SDK Platforms" "Version" = Get-AndroidPlatformVersions -PackageInfo $packageInfo - "Location" = Get-AndroidComponentLocation -ComponentName "platforms" }, @{ "Package" = "Android SDK Build-tools" "Version" = Get-AndroidBuildToolVersions -PackageInfo $packageInfo - "Location" = Get-AndroidComponentLocation -ComponentName "build-tools" }, @{ - "Package" = "Android SDK Platform-Tools" - "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Android SDK Platform-Tools" - }, - @{ - "Package" = "Google APIs" - "Version" = Get-AndroidGoogleAPIsVersions -PackageInfo $packageInfo + "Package" = "NDK" + "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "ndk-bundle" }, @{ "Package" = "Android Support Repository" "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Android Support Repository" }, + @{ + "Package" = "Google APIs" + "Version" = Get-AndroidGoogleAPIsVersions -PackageInfo $packageInfo + }, @{ "Package" = "Google Play services" "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "Google Play services" @@ -84,16 +86,11 @@ function Build-AndroidTable { @{ "Package" = "CMake" "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "cmake" - }, - @{ - "Package" = "NDK" - "Version" = Get-AndroidPackageVersions -PackageInfo $packageInfo -MatchedString "ndk-bundle" } ) | Where-Object { $_.Version } | ForEach-Object { [PSCustomObject] @{ "Package Name" = $_.Package "Version" = $_.Version - "Location" = $_.Location } } } From 9def9a1f02ace2f08543975f204522dd3139d56b Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 15 Sep 2020 15:25:32 +0300 Subject: [PATCH 14/31] Add "validate contributor permissions" step --- images.CI/macos/azure-pipelines/image-generation.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 8eb3f6998..5e42236c9 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -9,6 +9,17 @@ jobs: - group: Mac-Cloud Image Generation Key Vault steps: + - pwsh: | + $validСontributor = ${$env:CONTRIBUTOR_ALLOWLIST}.Split(",") | Where-Object { $_ -eq $env:BUILD_SOURCEVERSIONAUTHOR } ` + | Select-Object -First 1 + + if (-not $validСontributor) { + Write-Host "Failed to start this build. $env:BUILD_SOURCEVERSIONAUTHOR is an unknown contributor" + Write-Host "Please add $env:BUILD_SOURCEVERSIONAUTHOR to the allowed list to run this build" + exit 1 + } + displayName: Validate contributor permissions + - checkout: self clean: true fetchDepth: 1 From 80d5ed8f2a2e1159fccaf0386ee8d55a5f3e2963 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 15 Sep 2020 15:37:18 +0300 Subject: [PATCH 15/31] Add trim --- images.CI/macos/azure-pipelines/image-generation.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 5e42236c9..e453bf6f8 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -10,12 +10,12 @@ jobs: steps: - pwsh: | - $validСontributor = ${$env:CONTRIBUTOR_ALLOWLIST}.Split(",") | Where-Object { $_ -eq $env:BUILD_SOURCEVERSIONAUTHOR } ` - | Select-Object -First 1 + $validСontributor = ${$env:CONTRIBUTOR_ALLOWLIST}.Split(",").Trim() | Where-Object { $_ -eq $env:BUILD_SOURCEVERSIONAUTHOR } ` + | Select-Object -First 1 if (-not $validСontributor) { Write-Host "Failed to start this build. $env:BUILD_SOURCEVERSIONAUTHOR is an unknown contributor" - Write-Host "Please add $env:BUILD_SOURCEVERSIONAUTHOR to the allowed list to run this build" + Write-Host "Please add $env:BUILD_SOURCEVERSIONAUTHOR to the allowed list to run builds" exit 1 } displayName: Validate contributor permissions From 0b2ee087e36fdf165bc7c062d288953a06974845 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 15 Sep 2020 16:10:29 +0300 Subject: [PATCH 16/31] Fix comments --- images.CI/macos/azure-pipelines/image-generation.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index e453bf6f8..b650fe0ae 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -3,15 +3,16 @@ jobs: displayName: Image Generation (${{ parameters.image_label }}) timeoutInMinutes: 720 pool: - name: Mac-Cloud V2 Image Generation + name: Mac-Cloud Image Generation variables: - group: Mac-Cloud Image Generation - group: Mac-Cloud Image Generation Key Vault steps: - pwsh: | - $validСontributor = ${$env:CONTRIBUTOR_ALLOWLIST}.Split(",").Trim() | Where-Object { $_ -eq $env:BUILD_SOURCEVERSIONAUTHOR } ` - | Select-Object -First 1 + $allowedContributors = $env:CONTRIBUTOR_ALLOWLIST.Split(",").Trim() + $validСontributor = $allowedContributors | Where-Object { $_ -eq $env:BUILD_SOURCEVERSIONAUTHOR } ` + | Select-Object -First 1 if (-not $validСontributor) { Write-Host "Failed to start this build. $env:BUILD_SOURCEVERSIONAUTHOR is an unknown contributor" From 734baaa941acb59a0e6526aef4f32da8142c9b47 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 15 Sep 2020 19:14:07 +0300 Subject: [PATCH 17/31] Move credscan-exclusions file --- images.CI/{linux-and-win => }/credscan-exclusions.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename images.CI/{linux-and-win => }/credscan-exclusions.json (100%) diff --git a/images.CI/linux-and-win/credscan-exclusions.json b/images.CI/credscan-exclusions.json similarity index 100% rename from images.CI/linux-and-win/credscan-exclusions.json rename to images.CI/credscan-exclusions.json From 3d46b4867e4d0533d78ab271e22a02105ba0ec10 Mon Sep 17 00:00:00 2001 From: MaksimmZZZhukoff Date: Wed, 16 Sep 2020 13:27:13 +0300 Subject: [PATCH 18/31] Add script to validate contributor permissions --- .../azure-pipelines/image-generation.yml | 24 +++---- images.CI/macos/validate-contributor.ps1 | 63 +++++++++++++++++++ 2 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 images.CI/macos/validate-contributor.ps1 diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index b650fe0ae..5c2f562f1 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -9,22 +9,22 @@ jobs: - group: Mac-Cloud Image Generation Key Vault steps: - - pwsh: | - $allowedContributors = $env:CONTRIBUTOR_ALLOWLIST.Split(",").Trim() - $validСontributor = $allowedContributors | Where-Object { $_ -eq $env:BUILD_SOURCEVERSIONAUTHOR } ` - | Select-Object -First 1 - - if (-not $validСontributor) { - Write-Host "Failed to start this build. $env:BUILD_SOURCEVERSIONAUTHOR is an unknown contributor" - Write-Host "Please add $env:BUILD_SOURCEVERSIONAUTHOR to the allowed list to run builds" - exit 1 - } - displayName: Validate contributor permissions - - checkout: self clean: true fetchDepth: 1 + - task: PowerShell@2 + displayName: 'Validate contributor permissions' + condition: startsWith(variables['Build.SourceBranch'], 'refs/pull/') + inputs: + targetType: 'filePath' + filePath: ./images.CI/macos/validate-contributor.ps1 + pwsh: true + arguments: -RepositoryName "$(Build.Repository.Name)" ` + -AccessToken "$(GITHUB_FEED_TOKEN)" ` + -SourceBranch "$(Build.SourceBranch)" ` + -ContributorAllowList "$(CONTRIBUTOR_ALLOWLIST)" + - task: PowerShell@2 displayName: 'Download custom repository' condition: and(ne(variables['CUSTOM_REPOSITORY_URL'], ''), ne(variables['CUSTOM_REPOSITORY_BRANCH'], '')) diff --git a/images.CI/macos/validate-contributor.ps1 b/images.CI/macos/validate-contributor.ps1 new file mode 100644 index 000000000..312a9ef7e --- /dev/null +++ b/images.CI/macos/validate-contributor.ps1 @@ -0,0 +1,63 @@ +param( + [Parameter(Mandatory)] [string] $RepositoryName, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [string] $SourceBranch, + [Parameter(Mandatory)] [string] $ContributorAllowList +) + +function Build-AuthHeader { + param( + [Parameter(Mandatory)] [string] $AccessToken + ) + + $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("'':${AccessToken}")) + return "Basic ${base64AuthInfo}" +} + +function Get-PullRequest { + param( + [Parameter(Mandatory)] [string] $RepositoryName, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [UInt32] $PullRequestNumber + ) + + $requestUrl = "https://api.github.com/repos/$RepositoryName/pulls/$PullRequestNumber" + $authHeader = Build-AuthHeader -AccessToken $AccessToken + + $params = @{ + Method = "GET" + ContentType = "application/json" + Uri = $requestUrl + Headers = @{ Authorization = $authHeader } + } + + return Invoke-RestMethod @params +} + +function Validate-ContributorPermissions { + param( + [Parameter(Mandatory)] [string] $ContributorAllowList, + [Parameter(Mandatory)] [string] $ContributorName + ) + + $allowedContributors = $ContributorAllowList.Split(",").Trim() + $validСontributor = $allowedContributors | Where-Object { $_ -eq $ContributorName } ` + | Select-Object -First 1 + + if (-not $validСontributor) { + Write-Host "Failed to start this build. '$ContributorName' is an unknown contributor" + Write-Host "Please add '$ContributorName' to the allowed list to run builds" + exit 1 + } +} + +$pullRequestNumber = $SourceBranch.Split("/")[2] + +$pullRequestInfo = Get-PullRequest -RepositoryName $RepositoryName ` + -AccessToken $AccessToken ` + -PullRequestNumber $pullRequestNumber + +$contributorName = $pullRequestInfo.user.login + +Validate-ContributorPermissions -ContributorAllowList $ContributorAllowList ` + -ContributorName $contributorName \ No newline at end of file From 5dc70e7d03f6be2d8c3883e49f30d1c766d3f8a8 Mon Sep 17 00:00:00 2001 From: MaksimmZZZhukoff Date: Wed, 16 Sep 2020 14:36:05 +0300 Subject: [PATCH 19/31] Update variables in image-generation file --- .../azure-pipelines/image-generation.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 5c2f562f1..490cbda6d 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -21,7 +21,7 @@ jobs: filePath: ./images.CI/macos/validate-contributor.ps1 pwsh: true arguments: -RepositoryName "$(Build.Repository.Name)" ` - -AccessToken "$(GITHUB_FEED_TOKEN)" ` + -AccessToken "$(github-feed-token)" ` -SourceBranch "$(Build.SourceBranch)" ` -ContributorAllowList "$(CONTRIBUTOR_ALLOWLIST)" @@ -47,18 +47,18 @@ jobs: ) packer build -on-error=abort ` - -var="vcenter_server=$(vcenter_server_v2)" ` - -var="vcenter_username=$(vcenter_username_v2)" ` - -var="vcenter_password=$(vcenter_password_v2)" ` - -var="vcenter_datacenter=$(vcenter_datacenter_v2)" ` - -var="cluster_or_esxi_host=$(esxi_cluster_v2)" ` + -var="vcenter_server=$(vcenter-server-v2)" ` + -var="vcenter_username=$(vcenter-username-v2)" ` + -var="vcenter_password=$(vcenter-password-v2)" ` + -var="vcenter_datacenter=$(vcenter-datacenter-v2)" ` + -var="cluster_or_esxi_host=$(esxi-cluster-v2)" ` -var="esxi_datastore=${{ parameters.target_datastore }}" ` - -var="output_folder=$(output_folder)" ` - -var="vm_username=$(vm_username)" ` - -var="vm_password=$(vm_password)" ` + -var="output_folder=$(output-folder)" ` + -var="vm_username=$(vm-username)" ` + -var="vm_password=$(vm-password)" ` -var="build_id=$(Build.BuildNumber)" ` -var="baseimage_name=${{ parameters.base_image_name }}" ` - -var="github_feed_token=$(GITHUB_FEED_TOKEN)" ` + -var="github_feed_token=$(github-feed-token)" ` -var="xcode_install_user=$(xcode-installation-user)" ` -var="xcode_install_password=$(xcode-installation-password)" ` -color=false ` From 16486c520b68d02efe76edede1a0f741fb0700e8 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Thu, 17 Sep 2020 14:58:58 +0300 Subject: [PATCH 20/31] removed useless function --- .../scripts/SoftwareReport/SoftwareReport.Android.psm1 | 8 -------- 1 file changed, 8 deletions(-) diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 b/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 index d27ac6736..125613c38 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 @@ -5,14 +5,6 @@ function Split-TableRowByColumns { return $Row.Split("|") | ForEach-Object { $_.trim() } } -function Get-AndroidComponentLocation { - param( - [string] $ComponentName - ) - $path = Join-Path $env:ANDROID_HOME $ComponentName - return "Location $path" -} - function Get-AndroidSDKRoot { param( [string] $ComponentName From d5b409e8a2da15d8f45df7b36372072f3f2a1c71 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Thu, 17 Sep 2020 17:06:40 +0300 Subject: [PATCH 21/31] removed the comma --- images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 b/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 index 125613c38..f85423c2c 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Android.psm1 @@ -112,7 +112,7 @@ function Get-AndroidPlatformVersions { $packageInfoParts = Split-TableRowByColumns $_ $revision = $packageInfoParts[1] $version = $packageInfoParts[0].split(";")[1] - return "$version, (rev $revision)" + return "$version (rev $revision)" } [array]::Reverse($versions) return ($versions -Join "
") From a636448540ba83c39b0f0ddb4f180d47d1c27f7b Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Sun, 20 Sep 2020 19:53:35 +0300 Subject: [PATCH 22/31] hardcode edge --- images/win/scripts/Installers/Install-Edge.ps1 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/images/win/scripts/Installers/Install-Edge.ps1 b/images/win/scripts/Installers/Install-Edge.ps1 index ebb7871b7..ada5eaacb 100644 --- a/images/win/scripts/Installers/Install-Edge.ps1 +++ b/images/win/scripts/Installers/Install-Edge.ps1 @@ -17,12 +17,13 @@ Write-Host "Get the Microsoft Edge WebDriver version..." $RegistryPath = "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths" $EdgePath = (Get-ItemProperty "$RegistryPath\msedge.exe").'(default)' [version]$EdgeVersion = [System.Diagnostics.FileVersionInfo]::GetVersionInfo($EdgePath).ProductVersion -$EdgeDriverVersionUrl = "https://msedgedriver.azureedge.net/LATEST_RELEASE_$($EdgeVersion.Major)" +#$EdgeDriverVersionUrl = "https://msedgedriver.azureedge.net/LATEST_RELEASE_$($EdgeVersion.Major)" -$EdgeDriverVersionFile = Start-DownloadWithRetry -Url $EdgeDriverVersionUrl -Name "versioninfo.txt" -DownloadPath $EdgeDriverPath +#$EdgeDriverVersionFile = Start-DownloadWithRetry -Url $EdgeDriverVersionUrl -Name "versioninfo.txt" -DownloadPath $EdgeDriverPath +Add-Content -Path "${EdgeDriverPath}\versioninfo.txt" -Value "85.0.564.51" Write-Host "Download Microsoft Edge WebDriver..." -$EdgeDriverLatestVersion = Get-Content -Path $EdgeDriverVersionFile +$EdgeDriverLatestVersion = Get-Content -Path "${EdgeDriverPath}\versioninfo.txt" $EdgeDriverArchName = "edgedriver_win64.zip" $EdgeDriverDownloadUrl="https://msedgedriver.azureedge.net/${EdgeDriverLatestVersion}/${EdgeDriverArchName}" From 702796b09b922aaa91108eb83b4f7589391557be Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Mon, 21 Sep 2020 09:46:17 +0300 Subject: [PATCH 23/31] Bring latest changes from internal Mac-Cloud repository from 21 September --- images/macos/helpers/Xcode.Helpers.psm1 | 13 ++++++++ images/macos/provision/core/commonutils.sh | 15 ++++++--- images/macos/provision/core/dotnet.sh | 8 ++++- images/macos/provision/core/node.sh | 10 +++--- images/macos/provision/core/openssl.sh | 6 +++- images/macos/provision/core/python.sh | 3 ++ images/macos/provision/core/ruby.sh | 19 ++++++----- images/macos/provision/core/rubygem.sh | 16 +++++---- images/macos/provision/core/xcode-ctl.sh | 2 +- images/macos/provision/core/xcode-tools.sh | 14 +++++++- images/macos/provision/utils/xcode-utils.sh | 5 ++- .../SoftwareReport.Common.psm1 | 4 +-- .../SoftwareReport.Generator.ps1 | 17 ++++++---- .../software-report/SoftwareReport.Xcode.psm1 | 33 ++++++++++++------- images/macos/templates/macOS-11.0.json | 1 + images/macos/tests/Common.Tests.ps1 | 20 ++++++----- images/macos/tests/Python.Tests.ps1 | 6 ++-- images/macos/tests/Xamarin.Tests.ps1 | 2 +- images/macos/tests/Xcode.Tests.ps1 | 4 +-- images/macos/toolsets/toolset-10.15.json | 4 +-- images/macos/toolsets/toolset-11.0.json | 23 ++++--------- 21 files changed, 143 insertions(+), 82 deletions(-) diff --git a/images/macos/helpers/Xcode.Helpers.psm1 b/images/macos/helpers/Xcode.Helpers.psm1 index 6f3dcf730..dbc4803c2 100644 --- a/images/macos/helpers/Xcode.Helpers.psm1 +++ b/images/macos/helpers/Xcode.Helpers.psm1 @@ -91,4 +91,17 @@ function Get-XcodePairsList { $result += "$watchName $phoneName" } return $result +} + +function Test-XcodeStableVersion { + param([Parameter(Mandatory)][string]$Version) + + if ($Version -match "beta") { + return $false + } + if ($Version -match "GM") { + return $false + } + + return $true } \ No newline at end of file diff --git a/images/macos/provision/core/commonutils.sh b/images/macos/provision/core/commonutils.sh index efa4d0160..2f2a18ce5 100644 --- a/images/macos/provision/core/commonutils.sh +++ b/images/macos/provision/core/commonutils.sh @@ -1,15 +1,14 @@ #!/bin/sh - set -e source ~/utils/utils.sh + +# TO-DO: Move the list of brew packages and casks to toolset + # brew install binst_common_utils=( carthage - xctool cmake - bats - parallel subversion go gnupg @@ -27,6 +26,14 @@ binst_common_utils=( aria2 ) +if is_Less_BigSur; then + binst_common_utils+=( + xctool + bats + parallel + ) +fi + for package in ${binst_common_utils[@]}; do echo "Install $package" brew install $package diff --git a/images/macos/provision/core/dotnet.sh b/images/macos/provision/core/dotnet.sh index 4e8f7dfaa..780564881 100755 --- a/images/macos/provision/core/dotnet.sh +++ b/images/macos/provision/core/dotnet.sh @@ -19,7 +19,13 @@ chmod +x ./dotnet-install.sh ARGS_LIST=() echo "Parsing dotnet SDK (except rc and preview versions) from .json..." -if is_Less_Catalina; then +# TO-DO: move the list of versions to install to toolset +if is_BigSur; then + DOTNET_CHANNELS=( + 'https://raw.githubusercontent.com/dotnet/core/master/release-notes/2.1/releases.json' + 'https://raw.githubusercontent.com/dotnet/core/master/release-notes/3.1/releases.json' + ) +elif is_Less_Catalina; then DOTNET_CHANNELS=( 'https://raw.githubusercontent.com/dotnet/core/master/release-notes/2.1/releases.json' ) diff --git a/images/macos/provision/core/node.sh b/images/macos/provision/core/node.sh index f5bb099f7..daf5ea408 100644 --- a/images/macos/provision/core/node.sh +++ b/images/macos/provision/core/node.sh @@ -39,7 +39,9 @@ fi echo Installing yarn... curl -o- -L https://yarnpkg.com/install.sh | bash -for module in ${node_common_modules[@]}; do - echo "Install $module" - npm install -g $module -done +if is_Less_BigSur; then + for module in ${node_common_modules[@]}; do + echo "Install $module" + npm install -g $module + done +fi diff --git a/images/macos/provision/core/openssl.sh b/images/macos/provision/core/openssl.sh index 3b66441c4..0638dccf8 100755 --- a/images/macos/provision/core/openssl.sh +++ b/images/macos/provision/core/openssl.sh @@ -4,7 +4,11 @@ export PATH="/usr/local/bin:/usr/local/sbin:~/bin:$PATH" echo Installing OpenSSL... /usr/local/bin/brew install openssl -/usr/local/bin/brew link openssl --force + +if is_BigSur; then + ln -sf $(brew --prefix openssl)/bin/openssl /usr/local/bin/openssl + exit 0 +fi # Install OpenSSL 1.0.2t # https://www.openssl.org/policies/releasestrat.html - Version 1.0.2 will be supported until 2019-12-31 (LTS) diff --git a/images/macos/provision/core/python.sh b/images/macos/provision/core/python.sh index 05b135e1e..41488642e 100755 --- a/images/macos/provision/core/python.sh +++ b/images/macos/provision/core/python.sh @@ -1,3 +1,6 @@ +#!/bin/sh +source ~/utils/utils.sh + echo "Installing Python Tooling" echo "Brew Installing Python 3" diff --git a/images/macos/provision/core/ruby.sh b/images/macos/provision/core/ruby.sh index 912b00054..19877de3e 100755 --- a/images/macos/provision/core/ruby.sh +++ b/images/macos/provision/core/ruby.sh @@ -2,13 +2,16 @@ set -e -if [ $(id -u) -eq 0 ]; then - echo "This script can not run as root. Aborting..." - exit 1 +source ~/utils/utils.sh + +echo Installing Ruby... +if is_Less_BigSur; then + # We can't install latest ruby 2.7 as a default version related with bug + # https://github.com/fastlane/fastlane/issues/15397 + /usr/local/bin/brew install ruby@2.6 + ln -sf /usr/local/opt/ruby\@2.6 /usr/local/opt/ruby +else + brew install ruby fi -# We can't install latest ruby 2.7 as a default version related with bug -# https://github.com/fastlane/fastlane/issues/15397 -echo Installing Ruby... -/usr/local/bin/brew install ruby@2.6 -ln -sf /usr/local/opt/ruby\@2.6 /usr/local/opt/ruby + diff --git a/images/macos/provision/core/rubygem.sh b/images/macos/provision/core/rubygem.sh index 6872e57e7..83639e472 100755 --- a/images/macos/provision/core/rubygem.sh +++ b/images/macos/provision/core/rubygem.sh @@ -10,14 +10,16 @@ gem install xcode-install --force echo Installing CocoaPods... gem install cocoapods -# fix nomad-cli installation -if is_HighSierra; then - brew install libxml2 - gem install nokogiri -v 1.6.8.1 -- --use-system-libraries --with-xml2-include=$(brew --prefix libxml2)/include/libxml2 -fi +if is_Less_BigSur; then + # fix nomad-cli installation + if is_HighSierra; then + brew install libxml2 + gem install nokogiri -v 1.6.8.1 -- --use-system-libraries --with-xml2-include=$(brew --prefix libxml2)/include/libxml2 + fi -echo Installing nomad-cli... -gem install nomad-cli + echo Installing nomad-cli... + gem install nomad-cli +fi echo Installing xcpretty... gem install xcpretty diff --git a/images/macos/provision/core/xcode-ctl.sh b/images/macos/provision/core/xcode-ctl.sh index 18f352db0..cea7aa633 100644 --- a/images/macos/provision/core/xcode-ctl.sh +++ b/images/macos/provision/core/xcode-ctl.sh @@ -9,7 +9,7 @@ clt_placeholder="/tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress" touch $clt_placeholder clt_label=`/usr/sbin/softwareupdate -l | - grep -B 1 -E 'Command Line Tools' | + grep -B 1 -E 'Command Line Tools beta 5' | awk -F'*' '/^ *\\*/ {print \$2}' | sed -e 's/^ *Label: //' -e 's/^ *//' | sort -V | diff --git a/images/macos/provision/core/xcode-tools.sh b/images/macos/provision/core/xcode-tools.sh index f301a39c4..ef9bf648c 100755 --- a/images/macos/provision/core/xcode-tools.sh +++ b/images/macos/provision/core/xcode-tools.sh @@ -36,7 +36,12 @@ do extractXcodeXip $WORK_DIR "$VERSION_TO_INSTALL" # Remove "beta" postfix from version - XCODE_VERSION=$(echo $XCODE_VERSION | cut -d"_" -f 1) + if [[ $XCODE_VERSION == "12_beta" ]] && is_Catalina ; then + # trick to install Xcode 12 GM and Xcode 12 beta 6 side by side + XCODE_VERSION="12_beta" + else + XCODE_VERSION=$(echo $XCODE_VERSION | cut -d"_" -f 1) + fi echo "Checking if unpacked Xcode ${XCODE_VERSION} is valid" validateXcodeIntegrity "$WORK_DIR" @@ -69,6 +74,13 @@ do continue fi + if [[ $XCODE_VERSION == "12_beta" ]] && is_Catalina ; then + # trick to install Xcode 12 GM and Xcode 12 beta 6 side by side + XCODE_VERSION="12_beta" + else + XCODE_VERSION=$(echo $XCODE_VERSION | cut -d"_" -f 1) + fi + echo "Running 'runFirstLaunch' for Xcode ${XCODE_VERSION}..." runFirstLaunch $XCODE_VERSION done diff --git a/images/macos/provision/utils/xcode-utils.sh b/images/macos/provision/utils/xcode-utils.sh index 9cd795f03..0f7855ec1 100644 --- a/images/macos/provision/utils/xcode-utils.sh +++ b/images/macos/provision/utils/xcode-utils.sh @@ -11,7 +11,7 @@ getXcodeVersionToInstall() { local XCODE_VERSION="$1" if [[ ! $XCODE_VERSION =~ "_beta" ]]; then - echo "$XCODE_VERSION" + echo "${XCODE_VERSION//_/ }" else local XCODE_BETA="${XCODE_VERSION/_/ }" echo "$(xcversion list | sort -r | grep -m 1 "$XCODE_BETA")" @@ -59,7 +59,6 @@ extractXcodeXip() { local WORKING_DIR="$1" local XCODE_VERSION="$2" XCODE_XIP="${WORKING_DIR}/Xcode_${XCODE_VERSION// /_}.xip" - echo "XCODE_XIP = $XCODE_XIP" open -W $XCODE_XIP @@ -75,7 +74,7 @@ extractXcodeXip() { createBetaSymlink() { local XCODE_VERSION=$1 - if [[ $XCODE_VERSION =~ 1[01].* ]] || [[ $XCODE_VERSION == "12" ]] || [[ $XCODE_VERSION == "12_beta" ]]; then + if [[ $XCODE_VERSION =~ 1[01].* ]] || [[ $XCODE_VERSION == "12" ]]; then ln -sf "/Applications/Xcode_${XCODE_VERSION}.app" "/Applications/Xcode_${XCODE_VERSION}_beta.app" fi } \ No newline at end of file diff --git a/images/macos/software-report/SoftwareReport.Common.psm1 b/images/macos/software-report/SoftwareReport.Common.psm1 index cd3cce626..c3fe75171 100644 --- a/images/macos/software-report/SoftwareReport.Common.psm1 +++ b/images/macos/software-report/SoftwareReport.Common.psm1 @@ -58,7 +58,7 @@ function Get-GccVersion { $versionList = @("8", "9") $versionList | Foreach-Object { $version = Run-Command "gcc-${_} --version" | Select-Object -First 1 - "$version — available by ``gcc-${_}`` alias" + "$version - available by ``gcc-${_}`` alias" } } @@ -66,7 +66,7 @@ function Get-FortranVersion { $versionList = @("8", "9") $versionList | Foreach-Object { $version = Run-Command "gfortran-${_} --version" | Select-Object -First 1 - "$version — available by ``gfortran-${_}`` alias" + "$version - available by ``gfortran-${_}`` alias" } } diff --git a/images/macos/software-report/SoftwareReport.Generator.ps1 b/images/macos/software-report/SoftwareReport.Generator.ps1 index 9627e24e1..b8a5283cc 100644 --- a/images/macos/software-report/SoftwareReport.Generator.ps1 +++ b/images/macos/software-report/SoftwareReport.Generator.ps1 @@ -77,7 +77,6 @@ $homebrewVersion = Run-Command "brew --version" | Select-Object -First 1 $npmVersion = Run-Command "npm --version" $yarnVersion = Run-Command "yarn --version" $nugetVersion = Run-Command "nuget help" | Select-Object -First 1 | Take-Part -Part 2 -$pipVersion = Get-PipVersion -Version 2 $pip3Version = Get-PipVersion -Version 3 $condaVersion = Invoke-Expression "conda --version" $rubyGemsVersion = Run-Command "gem --version" @@ -88,7 +87,13 @@ if ($os.IsHigherThanMojave) { $vcpkgVersion = Get-VcpkgVersion $markdown += New-MDList -Lines $vcpkgVersion -Style Unordered -NoNewLine } +if ($os.IsLessThanBigSur) { + $pipVersion = Get-PipVersion -Version 2 + $markdown += New-MDList -Style Unordered -Lines @("Pip ${pipVersion}") -NoNewLine +} + $markdown += New-MDList -Style Unordered -Lines @( + "Pip ${pip3Version}", $bundlerVersion, "Carthage ${carthageVersion}", "CocoaPods ${cocoaPodsVersion}", @@ -96,8 +101,6 @@ $markdown += New-MDList -Style Unordered -Lines @( "NPM ${npmVersion}", "Yarn ${yarnVersion}", "NuGet ${nugetVersion}", - "Pip ${pipVersion}", - "Pip ${pip3Version}", "Mini${condaVersion}", "RubyGems ${rubyGemsVersion}", "Composer ${composerVersion}" @@ -124,7 +127,6 @@ $gitLFSVersion = Run-Command "git-lfs version" | Take-Part -Part 0 | Take-Part - $hubVersion = Run-Command "hub version | grep 'hub version'" | Take-Part -Part 2 $wgetVersion = Run-Command "wget --version" | Select-String "GNU Wget" | Take-Part -Part 2 $svnVersion = Run-Command "svn --version --quiet" -$parallelVersion = Run-Command "parallel --version" | Select-String "GNU parallel" | Select-Object -First 1 $jqVersion = Run-Command "jq --version" | Take-Part -Part 1 -Delimiter "-" $opensslVersion = Get-Item /usr/local/opt/openssl | ForEach-Object {"{0} ``({1} -> {2})``" -f (Run-Command "openssl version"), $_.FullName, $_.Target} $gpgVersion = Run-Command "gpg --version" | Select-String 'gpg (GnuPG)' -SimpleMatch @@ -153,7 +155,6 @@ $markdown += New-MDList -Style Unordered -NoNewLine -Lines @( "GNU Wget ${wgetVersion}", "Subversion (SVN) ${svnVersion}", "Packer $packerVersion", - $parallelVersion, $opensslVersion, "jq ${jqVersion}", $gpgVersion, @@ -173,7 +174,11 @@ $markdown += New-MDList -Style Unordered -NoNewLine -Lines @( ) if ($os.IsHigherThanMojave) { $newmanVersion = Run-Command "newman --version" - $markdown += New-MDList -Lines "Newman $newmanVersion" -Style Unordered + $markdown += New-MDList -Lines "Newman $newmanVersion" -Style Unordered -NoNewLine +} +if ($os.IsLessThanBigSur) { + $parallelVersion = Run-Command "parallel --version" | Select-String "GNU parallel" | Select-Object -First 1 + $markdown += New-MDList -Lines $parallelVersion -Style Unordered } $markdown += New-MDNewLine diff --git a/images/macos/software-report/SoftwareReport.Xcode.psm1 b/images/macos/software-report/SoftwareReport.Xcode.psm1 index d4cda2eb6..59754d583 100644 --- a/images/macos/software-report/SoftwareReport.Xcode.psm1 +++ b/images/macos/software-report/SoftwareReport.Xcode.psm1 @@ -1,3 +1,7 @@ +Import-Module "$PSScriptRoot/../helpers/Common.Helpers.psm1" + +$os = Get-OSVersion + function Get-XcodePaths { $xcodePaths = Get-ChildItemWithoutSymlinks "/Applications" -Filter "Xcode_*.app" return $xcodePaths | Select-Object -ExpandProperty Fullname @@ -43,7 +47,7 @@ function Get-XcodeInfoList { $versionInfo.Path = $xcodeRootPath $versionInfo.IsDefault = ($xcodeRootPath -eq $defaultXcodeRootPath) - $xcodeInfo.Add($versionInfo.Version.ToString(), [PSCustomObject] @{ + $xcodeInfo.Add($xcodeRootPath, [PSCustomObject] @{ VersionInfo = $versionInfo SDKInfo = Get-XcodeSDKList SimulatorsInfo = Get-XcodeSimulatorsInfo @@ -217,20 +221,27 @@ function Build-XcodeSimulatorsTable { } function Build-XcodeSupportToolsSection { - $nomadCLI = Run-Command "gem -v nomad-cli" - $nomadIPA = Run-Command "ipa -version" $xcpretty = Run-Command "xcpretty --version" - $xctool = Run-Command "xctool --version" $xcversion = Run-Command "xcversion --version" | Select-String "^[0-9]" + $toolList = @( + "xcpretty $xcpretty", + "xcversion $xcversion" + ) + + if ($os.IsLessThanBigSur) { + $nomadCLI = Run-Command "gem -v nomad-cli" + $nomadIPA = Run-Command "ipa -version" + $xctool = Run-Command "xctool --version" + $toolList += @( + "Nomad CLI $nomadCLI", + "Nomad CLI IPA $nomadIPA", + "xctool $xctool" + ) + } + $output = "" $output += New-MDHeader "Xcode Support Tools" -Level 4 - $output += New-MDList -Style Unordered -Lines @( - "Nomad CLI $nomadCLI", - "Nomad CLI IPA $nomadIPA", - "xcpretty $xcpretty", - "xctool $xctool", - "xcversion $xcversion" - ) + $output += New-MDList -Style Unordered -Lines $toolList return $output } \ No newline at end of file diff --git a/images/macos/templates/macOS-11.0.json b/images/macos/templates/macOS-11.0.json index 7cf7db8ed..6da9df38d 100644 --- a/images/macos/templates/macOS-11.0.json +++ b/images/macos/templates/macOS-11.0.json @@ -129,6 +129,7 @@ "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}", "pause_before": "30s", "scripts": [ + "./provision/core/xcode-ctl.sh", "./provision/core/homebrew.sh", "./provision/core/dotnet.sh", "./provision/core/python.sh", diff --git a/images/macos/tests/Common.Tests.ps1 b/images/macos/tests/Common.Tests.ps1 index ed813f58c..3e61a9a81 100644 --- a/images/macos/tests/Common.Tests.ps1 +++ b/images/macos/tests/Common.Tests.ps1 @@ -94,17 +94,19 @@ Describe "Common utilities" { "aliyun --version" | Should -ReturnZeroExitCode } - It "Mobile Center CLI" { + It "Mobile Center CLI" -Skip:($os.IsBigSur) { "mobile-center --version" | Should -ReturnZeroExitCode } - It "Nomad CLI" { - $result = Get-CommandResult "gem list" - $result.Output | Should -BeLike "*nomad-cli*" - } - - It "Nomad CLI IPA" { - "ipa --version" | Should -ReturnZeroExitCode + Context "Nomad" -Skip:($os.IsBigSur) { + It "Nomad CLI" { + $result = Get-CommandResult "gem list" + $result.Output | Should -BeLike "*nomad-cli*" + } + + It "Nomad CLI IPA" { + "ipa --version" | Should -ReturnZeroExitCode + } } It "Conda" { @@ -129,7 +131,7 @@ Describe "Common utilities" { "curl --version" | Should -ReturnZeroExitCode } - It "xctool" { + It "xctool" -Skip:($os.IsBigSur) { "xctool --version" | Should -ReturnZeroExitCode } diff --git a/images/macos/tests/Python.Tests.ps1 b/images/macos/tests/Python.Tests.ps1 index 7b5f8348d..76cd8941b 100644 --- a/images/macos/tests/Python.Tests.ps1 +++ b/images/macos/tests/Python.Tests.ps1 @@ -1,6 +1,8 @@ Import-Module "$PSScriptRoot/../helpers/Common.Helpers.psm1" Import-Module "$PSScriptRoot/../helpers/Tests.Helpers.psm1" +$os = Get-OSVersion + Describe "Python" { It "Python 2 is available" { "python --version" | Should -ReturnZeroExitCode @@ -10,7 +12,7 @@ Describe "Python" { (Get-CommandResult "python --version").Output | Should -BeLike "Python 2.*" } - It "Python 2 is installed under /usr/local/bin" { + It "Python 2 is installed under /usr/local/bin" -Skip:($os.IsBigSur) { Get-WhichTool "python" | Should -BeLike "/usr/local/bin*" } @@ -22,7 +24,7 @@ Describe "Python" { Get-WhichTool "python3" | Should -BeLike "/usr/local/bin*" } - It "Pip 2 is available" { + It "Pip 2 is available" -Skip:($os.IsBigSur) { "pip --version" | Should -ReturnZeroExitCode } diff --git a/images/macos/tests/Xamarin.Tests.ps1 b/images/macos/tests/Xamarin.Tests.ps1 index 5e60f5366..2145f29a7 100644 --- a/images/macos/tests/Xamarin.Tests.ps1 +++ b/images/macos/tests/Xamarin.Tests.ps1 @@ -191,7 +191,7 @@ Describe "Xamarin Bundles" { $XAMARIN_ANDROID_VERSIONS_PATH = "/Library/Frameworks/Xamarin.Android.framework/Versions" } - $XAMARIN_BUNDLES = Get-ToolsetValue "xamarin.bundles" + [array]$XAMARIN_BUNDLES = Get-ToolsetValue "xamarin.bundles" $XAMARIN_DEFAULT_BUNDLE = Get-ToolsetValue "xamarin.bundle-default" If ($XAMARIN_DEFAULT_BUNDLE -eq "latest") { $XAMARIN_DEFAULT_BUNDLE = $XAMARIN_BUNDLES[0].symlink } diff --git a/images/macos/tests/Xcode.Tests.ps1 b/images/macos/tests/Xcode.Tests.ps1 index 543a715db..ff8d5e932 100644 --- a/images/macos/tests/Xcode.Tests.ps1 +++ b/images/macos/tests/Xcode.Tests.ps1 @@ -56,7 +56,7 @@ Describe "Xcode" { } Context "XCODE_DEVELOPER_DIR" { - $stableXcodeVersions = $XCODE_VERSIONS | Where-Object { -not ($_ -match "beta") } + $stableXcodeVersions = $XCODE_VERSIONS | Where-Object { Test-XcodeStableVersion $_ } $majorXcodeVersions = $stableXcodeVersions | ForEach-Object { $_.Split(".")[0] } | Select-Object -Unique $testCases = $majorXcodeVersions | ForEach-Object { $majorXcodeVersion = $_ @@ -84,7 +84,7 @@ Describe "Xcode" { } Describe "Xcode simulators" { - $XCODE_VERSIONS | ForEach-Object { + $XCODE_VERSIONS | Where-Object { Test-XcodeStableVersion $_ } | ForEach-Object { Switch-Xcode -Version $_ Context "$_" { diff --git a/images/macos/toolsets/toolset-10.15.json b/images/macos/toolsets/toolset-10.15.json index c825865c7..e7b17989f 100644 --- a/images/macos/toolsets/toolset-10.15.json +++ b/images/macos/toolsets/toolset-10.15.json @@ -2,11 +2,11 @@ "xcode": { "default": "11.7", "versions": [ - "12_beta", "11.7", "11.6", "11.5", "11.4.1", "11.4", "11.3.1", "11.2.1", "11.1", "11", "10.3" + "12.2_beta", "12", "12_beta", "11.7", "11.6", "11.5", "11.4.1", "11.4", "11.3.1", "11.2.1", "11.1", "11", "10.3" ] }, "xamarin": { - "vsmac": "8.7.5.19", + "vsmac": "8.7.7.10", "mono-versions": [ "6.12.0.93", "6.10.0.106", "6.8.0.123", "6.6.0.166", "6.4.0.208" ], diff --git a/images/macos/toolsets/toolset-11.0.json b/images/macos/toolsets/toolset-11.0.json index bab2c6f92..fa7a65a5a 100644 --- a/images/macos/toolsets/toolset-11.0.json +++ b/images/macos/toolsets/toolset-11.0.json @@ -2,22 +2,22 @@ "xcode": { "default": "11.7", "versions": [ - "12_beta", "11.7" + "12.2_beta", "12_beta", "11.7" ] }, "xamarin": { - "vsmac": "8.7.5.19", + "vsmac": "8.7.7.10", "mono-versions": [ - "6.12.0.93", "6.10.0.106" + "6.12.0.93" ], "ios-versions": [ - "13.20.2.2", "13.18.2.1" + "13.20.2.2" ], "mac-versions": [ - "6.20.2.2", "6.18.3.2" + "6.20.2.2" ], "android-versions": [ - "11.0.2.0", "10.3.1.4" + "11.0.2.0" ], "bundle-default": "latest", "bundles": [ @@ -27,13 +27,6 @@ "ios": "13.20", "mac": "6.20", "android": "11.0" - }, - { - "symlink": "6_10_0", - "mono":"6.10", - "ios": "13.18", - "mac": "6.18", - "android": "10.3" } ] }, @@ -67,9 +60,6 @@ "arch": "x64", "platform" : "darwin", "versions": [ - "2.7.*", - "3.5.*", - "3.6.*", "3.7.*", "3.8.*" ] @@ -91,7 +81,6 @@ "arch": "x64", "platform" : "darwin", "versions": [ - "1.14.*", "1.15.*" ] } From 8de9bcc26c144ed1f31e874de40e98c7007e424f Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov Date: Mon, 21 Sep 2020 10:53:56 +0300 Subject: [PATCH 24/31] add AppleWWDRCAG3.cer --- .../configuration/configure-machine.sh | 11 +++++++++- images/macos/tests/Common.Tests.ps1 | 22 +++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/images/macos/provision/configuration/configure-machine.sh b/images/macos/provision/configuration/configure-machine.sh index d34d7002a..dbfae6731 100644 --- a/images/macos/provision/configuration/configure-machine.sh +++ b/images/macos/provision/configuration/configure-machine.sh @@ -18,4 +18,13 @@ sudo pmset hibernatemode 0 sudo rm -f /var/vm/sleepimage # Change screen resolution to the maximum supported for 4Mb video memory -sudo "/Library/Application Support/VMware Tools/vmware-resolutionSet" 1176 885 \ No newline at end of file +sudo "/Library/Application Support/VMware Tools/vmware-resolutionSet" 1176 885 + +# https://developer.apple.com/support/expiration/ +# Enterprise iOS Distribution Certificates generated between February 7 and September 1st, 2020 will expire on February 7, 2023. +# Rotate the certificate before expiration to ensure your apps are installed and signed with an active certificate. +# Confirm that the correct intermediate certificate is installed by verifying the expiration date is set to 2030. +# sudo security delete-certificate -Z FF6797793A3CD798DC5B2ABEF56F73EDC9F83A64 /Library/Keychains/System.keychain +curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer --output $HOME/AppleWWDRCAG3.cer --silent +sudo security add-trusted-cert -d -r unspecified -k /Library/Keychains/System.keychain $HOME/AppleWWDRCAG3.cer +rm $HOME/AppleWWDRCAG3.cer \ No newline at end of file diff --git a/images/macos/tests/Common.Tests.ps1 b/images/macos/tests/Common.Tests.ps1 index ed813f58c..df7d505e5 100644 --- a/images/macos/tests/Common.Tests.ps1 +++ b/images/macos/tests/Common.Tests.ps1 @@ -12,6 +12,14 @@ Describe "Disk free space" { } } +Describe "Certificate" { + It "Apple Worldwide Developer Relations Certification Authority[expired: 2030-02] is installed" { + $sha1Hash = "06EC06599F4ED0027CC58956B4D3AC1255114F35" + $certs = security find-certificate -a -c Worldwide -p -Z | Out-String + $certs | Should -Match $sha1Hash + } +} + Describe "Git" { It "git is installed" { "git --version" | Should -ReturnZeroExitCode @@ -161,7 +169,7 @@ Describe "Common utilities" { It "PostgreSQL-Client" { "psql --version" | Should -ReturnZeroExitCode } - + It "PostgreSQL-Server" { "pg_config --version" | Should -ReturnZeroExitCode } @@ -178,11 +186,11 @@ Describe "Common utilities" { Get-WhichTool "php" | Should -Not -BeLike "/usr/bin/php*" "php --version" | Should -ReturnZeroExitCode } - + It "Composer" { "composer --version" | Should -ReturnZeroExitCode } - + It "R" -Skip:($os.IsBigSur) { "R --version" | Should -ReturnZeroExitCode } @@ -198,7 +206,7 @@ Describe "Common utilities" { It "bazelisk" { "bazelisk version" | Should -ReturnZeroExitCode } - + It "Julia" { "julia --version" | Should -ReturnZeroExitCode } @@ -252,7 +260,7 @@ Describe "Browsers" { It "Microsoft Edge Driver" { "msedgedriver --version" | Should -ReturnZeroExitCode } - + It "Firefox" { $firefoxLocation = "/Applications/Firefox.app/Contents/MacOS/firefox" $firefoxLocation | Should -Exist @@ -304,7 +312,7 @@ Describe "Haskell" -Skip:($os.IsHighSierra) { It "GHC" { "ghc --version" | Should -ReturnZeroExitCode } - + It "Cabal" { "cabal --version" | Should -ReturnZeroExitCode } @@ -327,7 +335,7 @@ Describe "Gcc" -Skip:($os.IsHighSierra) { param ( [string] $GccVersion ) - + "gcc-$GccVersion --version" | Should -ReturnZeroExitCode } } From d4335cc52e0569b7e5ac9720b10ddbecaede0995 Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov Date: Mon, 21 Sep 2020 11:18:28 +0300 Subject: [PATCH 25/31] set chmod -R 777 /opt --- .../scripts/installers/post-deployment.sh | 11 ++++++++ images/linux/ubuntu1604.json | 25 ++++++++++++------- images/linux/ubuntu1804.json | 25 ++++++++++++------- images/linux/ubuntu2004.json | 7 ++++++ 4 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 images/linux/scripts/installers/post-deployment.sh diff --git a/images/linux/scripts/installers/post-deployment.sh b/images/linux/scripts/installers/post-deployment.sh new file mode 100644 index 000000000..10fe34feb --- /dev/null +++ b/images/linux/scripts/installers/post-deployment.sh @@ -0,0 +1,11 @@ +#!/bin/bash +################################################################################ +## File: post-deployment.sh +## Desc: Post deployment actions +################################################################################ + +# set chmod -R 777 /opt +if [[ -d "/opt" ]]; then + echo "chmod -R 777 /opt" + chmod -R 777 /opt +fi diff --git a/images/linux/ubuntu1604.json b/images/linux/ubuntu1604.json index 33263345a..111f87f0d 100644 --- a/images/linux/ubuntu1604.json +++ b/images/linux/ubuntu1604.json @@ -301,15 +301,6 @@ "destination": "{{template_dir}}/Ubuntu1604-README.md", "direction": "download" }, - { - "type": "shell", - "inline": [ - "rm -rf {{user `helper_script_folder`}}", - "rm -rf {{user `installer_script_folder`}}", - "chmod 755 {{user `image_folder`}}" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, { "type": "shell", "scripts":[ @@ -319,6 +310,22 @@ "RUN_VALIDATION={{user `run_validation_diskspace`}}" ] }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/post-deployment.sh" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, + { + "type": "shell", + "inline": [ + "rm -rf {{user `helper_script_folder`}}", + "rm -rf {{user `installer_script_folder`}}", + "chmod 755 {{user `image_folder`}}" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, { "type": "file", "source": "{{template_dir}}/config/ubuntu1604.conf", diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index 1dd62bc2d..d62de1e48 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -305,15 +305,6 @@ "destination": "{{template_dir}}/Ubuntu1804-README.md", "direction": "download" }, - { - "type": "shell", - "inline": [ - "rm -rf {{user `helper_script_folder`}}", - "rm -rf {{user `installer_script_folder`}}", - "chmod 755 {{user `image_folder`}}" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, { "type": "shell", "scripts":[ @@ -323,6 +314,22 @@ "RUN_VALIDATION={{user `run_validation_diskspace`}}" ] }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/post-deployment.sh" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, + { + "type": "shell", + "inline": [ + "rm -rf {{user `helper_script_folder`}}", + "rm -rf {{user `installer_script_folder`}}", + "chmod 755 {{user `image_folder`}}" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, { "type": "file", "source": "{{template_dir}}/config/ubuntu1804.conf", diff --git a/images/linux/ubuntu2004.json b/images/linux/ubuntu2004.json index 67d6e1d6a..5a0f77609 100644 --- a/images/linux/ubuntu2004.json +++ b/images/linux/ubuntu2004.json @@ -316,6 +316,13 @@ "RUN_VALIDATION={{user `run_validation_diskspace`}}" ] }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/post-deployment.sh" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, { "type": "shell", "inline": [ From 6986bb8008905424d8c2b140323b841a783f97a8 Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Tue, 22 Sep 2020 10:44:51 +0300 Subject: [PATCH 26/31] set condition to install .51 instead of .60 --- images/win/scripts/Installers/Install-Edge.ps1 | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/images/win/scripts/Installers/Install-Edge.ps1 b/images/win/scripts/Installers/Install-Edge.ps1 index ada5eaacb..888a9285e 100644 --- a/images/win/scripts/Installers/Install-Edge.ps1 +++ b/images/win/scripts/Installers/Install-Edge.ps1 @@ -17,15 +17,21 @@ Write-Host "Get the Microsoft Edge WebDriver version..." $RegistryPath = "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths" $EdgePath = (Get-ItemProperty "$RegistryPath\msedge.exe").'(default)' [version]$EdgeVersion = [System.Diagnostics.FileVersionInfo]::GetVersionInfo($EdgePath).ProductVersion -#$EdgeDriverVersionUrl = "https://msedgedriver.azureedge.net/LATEST_RELEASE_$($EdgeVersion.Major)" +$EdgeDriverVersionUrl = "https://msedgedriver.azureedge.net/LATEST_RELEASE_$($EdgeVersion.Major)" -#$EdgeDriverVersionFile = Start-DownloadWithRetry -Url $EdgeDriverVersionUrl -Name "versioninfo.txt" -DownloadPath $EdgeDriverPath -Add-Content -Path "${EdgeDriverPath}\versioninfo.txt" -Value "85.0.564.51" +$EdgeDriverVersionFile = Start-DownloadWithRetry -Url $EdgeDriverVersionUrl -Name "versioninfo.txt" -DownloadPath $EdgeDriverPath Write-Host "Download Microsoft Edge WebDriver..." $EdgeDriverLatestVersion = Get-Content -Path "${EdgeDriverPath}\versioninfo.txt" $EdgeDriverArchName = "edgedriver_win64.zip" -$EdgeDriverDownloadUrl="https://msedgedriver.azureedge.net/${EdgeDriverLatestVersion}/${EdgeDriverArchName}" +# A temporary workaround to install the previous driver version because 85.0.564.60 for win64 doesn't exist +if ($EdgeDriverLatestVersion -eq "85.0.564.60") +{ + $EdgeDriverLatestVersion = "85.0.564.51" + Set-Content -Path "${EdgeDriverPath}\versioninfo.txt" -Value $EdgeDriverLatestVersion +} + +$EdgeDriverDownloadUrl = "https://msedgedriver.azureedge.net/${EdgeDriverLatestVersion}/${EdgeDriverArchName}" $EdgeDriverArchPath = Start-DownloadWithRetry -Url $EdgeDriverDownloadUrl -Name $EdgeDriverArchName From 3a47b604fd3117c9db6d99fcd0ffc74dd263e0c2 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Tue, 22 Sep 2020 09:46:15 +0200 Subject: [PATCH 27/31] fixes #1633 , using azure module, adjusted documentation --- help/CreateImageAndAzureResources.md | 3 ++- helpers/GenerateResourcesAndImage.ps1 | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/help/CreateImageAndAzureResources.md b/help/CreateImageAndAzureResources.md index 123d335cf..3401c1fce 100644 --- a/help/CreateImageAndAzureResources.md +++ b/help/CreateImageAndAzureResources.md @@ -12,6 +12,7 @@ After successful image generation, a snapshot of the temporary VM will be conver - `packer` - Can be downloaded from https://www.packer.io/downloads - `PowerShell 5.0 or higher` or `PSCore` for linux distributes. - `Azure CLI ` - https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest +- `Azure Powershell module` - https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-4.6.1 ### Azure DevOps self-hosted pool requirements To connect to a temporary VM packer use WinRM or SSH connections on public IP interfaces. @@ -31,7 +32,7 @@ Download `packer` from https://www.packer.io/downloads, or install it via Chocol choco install packer ``` -Install Azure CLI - https://docs.microsoft.com/ru-ru/cli/azure/install-azure-cli-windows?view=azure-cli-latest&tabs=azure-cli. +Install Azure CLI - https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-windows?view=azure-cli-latest&tabs=azure-cli. ``` Invoke-WebRequest -Uri https://aka.ms/installazurecliwindows -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; rm .\AzureCLI.msi ``` diff --git a/helpers/GenerateResourcesAndImage.ps1 b/helpers/GenerateResourcesAndImage.ps1 index 321770949..31d48ee65 100644 --- a/helpers/GenerateResourcesAndImage.ps1 +++ b/helpers/GenerateResourcesAndImage.ps1 @@ -110,12 +110,12 @@ Function GenerateResourcesAndImage { $ServicePrincipalClientSecret = $env:UserName + [System.GUID]::NewGuid().ToString().ToUpper(); $InstallPassword = $env:UserName + [System.GUID]::NewGuid().ToString().ToUpper(); - Login-AzureRmAccount - Set-AzureRmContext -SubscriptionId $SubscriptionId + Connect-AzAccount + Set-AzContext -SubscriptionId $SubscriptionId $alreadyExists = $true; try { - Get-AzureRmResourceGroup -Name $ResourceGroupName + Get-AzResourceGroup -Name $ResourceGroupName Write-Verbose "Resource group was found, will delete and recreate it." } catch { @@ -126,8 +126,8 @@ Function GenerateResourcesAndImage { if ($alreadyExists) { if($Force -eq $true) { # Cleanup the resource group if it already exitsted before - Remove-AzureRmResourceGroup -Name $ResourceGroupName -Force - New-AzureRmResourceGroup -Name $ResourceGroupName -Location $AzureLocation + Remove-AzResourceGroup -Name $ResourceGroupName -Force + New-AzResourceGroup -Name $ResourceGroupName -Location $AzureLocation } else { $title = "Delete Resource Group" $message = "The resource group you specified already exists. Do you want to clean it up?" @@ -146,13 +146,13 @@ Function GenerateResourcesAndImage { switch ($result) { - 0 { Remove-AzureRmResourceGroup -Name $ResourceGroupName -Force; New-AzureRmResourceGroup -Name $ResourceGroupName -Location $AzureLocation } + 0 { Remove-AzResourceGroup -Name $ResourceGroupName -Force; New-AzResourceGroup -Name $ResourceGroupName -Location $AzureLocation } 1 { <# Do nothing #> } 2 { exit } } } } else { - New-AzureRmResourceGroup -Name $ResourceGroupName -Location $AzureLocation + New-AzResourceGroup -Name $ResourceGroupName -Location $AzureLocation } # This script should follow the recommended naming conventions for azure resources @@ -164,19 +164,19 @@ Function GenerateResourcesAndImage { $storageAccountName = $storageAccountName.Replace("-", "").Replace("_", "").Replace("(", "").Replace(")", "").ToLower() $storageAccountName += "001" - New-AzureRmStorageAccount -ResourceGroupName $ResourceGroupName -AccountName $storageAccountName -Location $AzureLocation -SkuName "Standard_LRS" + New-AzStorageAccount -ResourceGroupName $ResourceGroupName -AccountName $storageAccountName -Location $AzureLocation -SkuName "Standard_LRS" $spDisplayName = [System.GUID]::NewGuid().ToString().ToUpper() - $sp = New-AzureRmADServicePrincipal -DisplayName $spDisplayName -Password (ConvertTo-SecureString $ServicePrincipalClientSecret -AsPlainText -Force) + $sp = New-AzADServicePrincipal -DisplayName $spDisplayName -Password (ConvertTo-SecureString $ServicePrincipalClientSecret -AsPlainText -Force) $spAppId = $sp.ApplicationId $spClientId = $sp.ApplicationId $spObjectId = $sp.Id Start-Sleep -Seconds $SecondsToWaitForServicePrincipalSetup - New-AzureRmRoleAssignment -RoleDefinitionName Contributor -ServicePrincipalName $spAppId + New-AzRoleAssignment -RoleDefinitionName Contributor -ServicePrincipalName $spAppId Start-Sleep -Seconds $SecondsToWaitForServicePrincipalSetup - $sub = Get-AzureRmSubscription -SubscriptionId $SubscriptionId + $sub = Get-AzSubscription -SubscriptionId $SubscriptionId $tenantId = $sub.TenantId # "", "Note this variable-setting script for running Packer with these Azure resources in the future:", "==============================================================================================", "`$spClientId = `"$spClientId`"", "`$ServicePrincipalClientSecret = `"$ServicePrincipalClientSecret`"", "`$SubscriptionId = `"$SubscriptionId`"", "`$tenantId = `"$tenantId`"", "`$spObjectId = `"$spObjectId`"", "`$AzureLocation = `"$AzureLocation`"", "`$ResourceGroupName = `"$ResourceGroupName`"", "`$storageAccountName = `"$storageAccountName`"", "`$install_password = `"$install_password`"", "" From e5a5a6cc39933885987c09767de50d255dfba228 Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Tue, 22 Sep 2020 10:50:32 +0300 Subject: [PATCH 28/31] fix path --- images/win/scripts/Installers/Install-Edge.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/win/scripts/Installers/Install-Edge.ps1 b/images/win/scripts/Installers/Install-Edge.ps1 index 888a9285e..1f4ce0ae4 100644 --- a/images/win/scripts/Installers/Install-Edge.ps1 +++ b/images/win/scripts/Installers/Install-Edge.ps1 @@ -22,13 +22,13 @@ $EdgeDriverVersionUrl = "https://msedgedriver.azureedge.net/LATEST_RELEASE_$($Ed $EdgeDriverVersionFile = Start-DownloadWithRetry -Url $EdgeDriverVersionUrl -Name "versioninfo.txt" -DownloadPath $EdgeDriverPath Write-Host "Download Microsoft Edge WebDriver..." -$EdgeDriverLatestVersion = Get-Content -Path "${EdgeDriverPath}\versioninfo.txt" +$EdgeDriverLatestVersion = Get-Content -Path $EdgeDriverVersionFile $EdgeDriverArchName = "edgedriver_win64.zip" # A temporary workaround to install the previous driver version because 85.0.564.60 for win64 doesn't exist if ($EdgeDriverLatestVersion -eq "85.0.564.60") { $EdgeDriverLatestVersion = "85.0.564.51" - Set-Content -Path "${EdgeDriverPath}\versioninfo.txt" -Value $EdgeDriverLatestVersion + Set-Content -Path $EdgeDriverVersionFile -Value $EdgeDriverLatestVersion } $EdgeDriverDownloadUrl = "https://msedgedriver.azureedge.net/${EdgeDriverLatestVersion}/${EdgeDriverArchName}" From 0d663601b655b25824db345b0b20413bbd512a31 Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Tue, 22 Sep 2020 11:25:51 +0300 Subject: [PATCH 29/31] Check environment variables --- images.CI/macos/azure-pipelines/image-generation.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 490cbda6d..4798c73fa 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -13,6 +13,10 @@ jobs: clean: true fetchDepth: 1 + - pwsh: | + Get-ChildItem Env: + exit 1 + - task: PowerShell@2 displayName: 'Validate contributor permissions' condition: startsWith(variables['Build.SourceBranch'], 'refs/pull/') From 9a48139d0e372c299f06972b43d035f91a9550c3 Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Tue, 22 Sep 2020 12:54:47 +0300 Subject: [PATCH 30/31] add attempts --- images.CI/macos/azure-pipelines/image-generation.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 4798c73fa..7f02d90f9 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -7,16 +7,14 @@ jobs: variables: - group: Mac-Cloud Image Generation - group: Mac-Cloud Image Generation Key Vault + - name: VirtualMachineName + value: $(Build.BuildNumber).$(System.JobAttempt) steps: - checkout: self clean: true fetchDepth: 1 - - pwsh: | - Get-ChildItem Env: - exit 1 - - task: PowerShell@2 displayName: 'Validate contributor permissions' condition: startsWith(variables['Build.SourceBranch'], 'refs/pull/') @@ -60,7 +58,7 @@ jobs: -var="output_folder=$(output-folder)" ` -var="vm_username=$(vm-username)" ` -var="vm_password=$(vm-password)" ` - -var="build_id=$(Build.BuildNumber)" ` + -var="build_id=${{ variables.VirtualMachineName }}" ` -var="baseimage_name=${{ parameters.base_image_name }}" ` -var="github_feed_token=$(github-feed-token)" ` -var="xcode_install_user=$(xcode-installation-user)" ` @@ -87,7 +85,7 @@ jobs: ls $(Common.TestResultsDirectory) echo "Put VM name to 'VM_Done_Name' file" - echo "$(Build.BuildNumber)" > "$(Build.ArtifactStagingDirectory)/VM_Done_Name" + echo "${{ variables.VirtualMachineName }}" > "$(Build.ArtifactStagingDirectory)/VM_Done_Name" displayName: Prepare artifact - bash: | From fe9f74e51ed37a716f61a7e9de6286bf57ef55c8 Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov Date: Tue, 22 Sep 2020 13:55:21 +0300 Subject: [PATCH 31/31] rm helper and installer folder in post-deployment --- .../scripts/installers/post-deployment.sh | 5 ++++ images/linux/ubuntu1604.json | 28 ++++++++----------- images/linux/ubuntu1804.json | 28 ++++++++----------- images/linux/ubuntu2004.json | 28 ++++++++----------- 4 files changed, 41 insertions(+), 48 deletions(-) diff --git a/images/linux/scripts/installers/post-deployment.sh b/images/linux/scripts/installers/post-deployment.sh index 10fe34feb..2864dae4f 100644 --- a/images/linux/scripts/installers/post-deployment.sh +++ b/images/linux/scripts/installers/post-deployment.sh @@ -9,3 +9,8 @@ if [[ -d "/opt" ]]; then echo "chmod -R 777 /opt" chmod -R 777 /opt fi + +# remove installer and helper folders +rm -rf $HELPER_SCRIPT_FOLDER +rm -rf $INSTALLER_SCRIPT_FOLDER +chmod 755 $IMAGE_FOLDER diff --git a/images/linux/ubuntu1604.json b/images/linux/ubuntu1604.json index 111f87f0d..3aa477cb6 100644 --- a/images/linux/ubuntu1604.json +++ b/images/linux/ubuntu1604.json @@ -301,6 +301,18 @@ "destination": "{{template_dir}}/Ubuntu1604-README.md", "direction": "download" }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/post-deployment.sh" + ], + "environment_vars":[ + "HELPER_SCRIPT_FOLDER={{user `helper_script_folder`}}", + "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}", + "IMAGE_FOLDER={{user `image_folder`}}" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, { "type": "shell", "scripts":[ @@ -310,22 +322,6 @@ "RUN_VALIDATION={{user `run_validation_diskspace`}}" ] }, - { - "type": "shell", - "scripts":[ - "{{template_dir}}/scripts/installers/post-deployment.sh" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, - { - "type": "shell", - "inline": [ - "rm -rf {{user `helper_script_folder`}}", - "rm -rf {{user `installer_script_folder`}}", - "chmod 755 {{user `image_folder`}}" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, { "type": "file", "source": "{{template_dir}}/config/ubuntu1604.conf", diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index d62de1e48..a5b71c07b 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -305,6 +305,18 @@ "destination": "{{template_dir}}/Ubuntu1804-README.md", "direction": "download" }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/post-deployment.sh" + ], + "environment_vars":[ + "HELPER_SCRIPT_FOLDER={{user `helper_script_folder`}}", + "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}", + "IMAGE_FOLDER={{user `image_folder`}}" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, { "type": "shell", "scripts":[ @@ -314,22 +326,6 @@ "RUN_VALIDATION={{user `run_validation_diskspace`}}" ] }, - { - "type": "shell", - "scripts":[ - "{{template_dir}}/scripts/installers/post-deployment.sh" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, - { - "type": "shell", - "inline": [ - "rm -rf {{user `helper_script_folder`}}", - "rm -rf {{user `installer_script_folder`}}", - "chmod 755 {{user `image_folder`}}" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, { "type": "file", "source": "{{template_dir}}/config/ubuntu1804.conf", diff --git a/images/linux/ubuntu2004.json b/images/linux/ubuntu2004.json index 5a0f77609..c2a1bd561 100644 --- a/images/linux/ubuntu2004.json +++ b/images/linux/ubuntu2004.json @@ -307,6 +307,18 @@ "destination": "{{template_dir}}/Ubuntu2004-README.md", "direction": "download" }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/post-deployment.sh" + ], + "environment_vars":[ + "HELPER_SCRIPT_FOLDER={{user `helper_script_folder`}}", + "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}", + "IMAGE_FOLDER={{user `image_folder`}}" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, { "type": "shell", "scripts":[ @@ -316,22 +328,6 @@ "RUN_VALIDATION={{user `run_validation_diskspace`}}" ] }, - { - "type": "shell", - "scripts":[ - "{{template_dir}}/scripts/installers/post-deployment.sh" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, - { - "type": "shell", - "inline": [ - "rm -rf {{user `helper_script_folder`}}", - "rm -rf {{user `installer_script_folder`}}", - "chmod 755 {{user `image_folder`}}" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, { "type": "file", "source": "{{template_dir}}/config/ubuntu2004.conf",