[macos] refactor XCode installation approach (#7858)

* [macos] refactor XCode installation approach

xcversion does not work anymore, we are switching to storing
XCode installers in intermediate Azure storage

* remove xcode_install_user, xcode_install_password

* rename xcode_install_storage --> xcode_install_storage_url

* mark xcode installation variables sensitive

* remove xcversion_auth_cookie variable

* renamed forgotten xcode_install_storage --> xcode_install_storage_url

* remove leftover xcode_install_user

.github/workflows/macos-generation.yml

@@ -105,13 +105,6 @@ jobs:
           -VIPassword ${{ secrets.VI_PASSWORD }} `
           -Cluster ${{ env.ESXI_CLUSTER }}
 
-    - name: Create xcversion session cookie file
-      shell: bash
-      run: |
-        mkdir -p ${{ runner.temp }}/xcversion-cookie
-        cookie='${{ secrets.XCVERSION_AUTH_COOKIE }}'
-        echo "$cookie" > ${{ runner.temp }}/xcversion-cookie/cookie
-
     - name: Build VM
       run: |
         $SensitiveData = @(
@@ -131,9 +124,6 @@ jobs:
         -var="github_api_pat=${{ secrets.GH_FEED_TOKEN }}" `
         -var="build_id=${{ env.VM_NAME }}" `
         -var="baseimage_name=${{ inputs.base_image_name }}" `
-        -var="xcode_install_user=${{ secrets.XCODE_USER }}" `
-        -var="xcode_install_password=${{ secrets.XCODE_PASSWORD }}" `
-        -var="xcversion_auth_cookie=${{ env.XCVERSION_COOKIE_PATH }}" `
         -color=false `
         ${{ inputs.template_path }} `
         | Where-Object {

images.CI/macos/azure-pipelines/image-generation.yml

@@ -83,9 +83,8 @@ jobs:
         -var="github_api_pat=$(github_api_pat)" `
         -var="build_id=$(VirtualMachineName)" `
         -var="baseimage_name=${{ parameters.base_image_name }}" `
-        -var="xcode_install_user=$(xcode-installation-user)" `
+        -var="xcode_install_storage_url=$(xcode_install_storage_url)" `
-        -var="xcode_install_password=$(xcode-installation-password)" `
+        -var="xcode_install_sas=$(xcode_install_sas)" `
-        -var="xcversion_auth_cookie=$(xcVersionCookie.secureFilePath)" `
         -color=false `
         ${{ parameters.template_path }} `
         | Where-Object {

images/macos/helpers/Xcode.Installer.psm1

@@ -24,17 +24,14 @@ function Invoke-DownloadXcodeArchive {
         [string]$Version
     )
 
-    $resolvedVersion = Resolve-ExactXcodeVersion -Version $Version
+    Write-Host "Downloading Xcode $Version"
-    if (-not $resolvedVersion) {
-        throw "Version '$Version' can't be matched to any available version"
-    }
-    Write-Host "Downloading Xcode $resolvedVersion"
-    Invoke-XCVersion -Arguments "install '$resolvedVersion' --no-install" | Out-Host    
 
-    $xcodeXipName = "$resolvedVersion" -replace " ", "_"
+    $tempXipDirectory = New-Item -Path $DownloadDirectory -Name "Xcode$Version" -ItemType "Directory"
-    $xcodeXipFile = Get-ChildItem -Path $DownloadDirectory -Filter "Xcode_$xcodeXipName.xip" | Select-Object -First 1
+
-    $tempXipDirectory = New-Item -Path $DownloadDirectory -Name "Xcode$xcodeXipName" -ItemType "Directory"
+    $xcodeFileName = 'Xcode-{0}.xip' -f $Version
-    Move-Item -Path "$xcodeXipFile" -Destination $tempXipDirectory
+    $xcodeUri = '{0}{1}{2}'-f ${env:XCODE_INSTALL_STORAGE_URL}, $xcodeFileName, ${env:XCODE_INSTALL_SAS}
+
+    Invoke-WebRequest -Uri $xcodeUri -OutFile (Join-Path $tempXipDirectory $xcodeFileName)
 
     return $tempXipDirectory
 
@@ -86,7 +83,7 @@ function Expand-XcodeXipArchive {
         [string]$TargetPath
     )
 
-    $xcodeXipPath = Get-ChildItem -Path $DownloadDirectory -Filter "Xcode_*.xip" | Select-Object -First 1
+    $xcodeXipPath = Get-ChildItem -Path $DownloadDirectory -Filter "Xcode-*.xip" | Select-Object -First 1
 
     Write-Host "Extracting Xcode from '$xcodeXipPath'"
     Push-Location $DownloadDirectory

images/macos/provision/core/xcode.ps1

@@ -1,15 +1,8 @@
-# The script currently requires 2 external variables to be set: XCODE_INSTALL_USER
-# and XCODE_INSTALL_PASSWORD, in order to access the Apple Developer Center
-
 $ErrorActionPreference = "Stop"
 
 Import-Module "$env:HOME/image-generation/helpers/Common.Helpers.psm1"
 Import-Module "$env:HOME/image-generation/helpers/Xcode.Installer.psm1"
 
-if ([string]::IsNullOrEmpty($env:XCODE_INSTALL_USER) -or [string]::IsNullOrEmpty($env:XCODE_INSTALL_PASSWORD)) {
-    throw "Required environment variables XCODE_INSTALL_USER and XCODE_INSTALL_PASSWORD are not set"
-}
-
 # Spaceship Apple ID login fails due to Apple ID prompting to be upgraded to 2FA.
 # https://github.com/fastlane/fastlane/pull/18116
 $env:SPACESHIP_SKIP_2FA_UPGRADE = 1

images/macos/templates/macOS-11.anka.pkr.hcl

@@ -34,21 +34,16 @@ variable "github_api_pat" {
   default = ""
 }
 
-variable "xcode_install_user" {
+variable "xcode_install_storage_url" {
   type = string
   sensitive = true
 }
 
-variable "xcode_install_password" {
+variable "xcode_install_sas" {
   type = string
   sensitive = true
 }
 
-variable "xcversion_auth_cookie" {
-  type = string
-  default = ""
-}
-
 variable "vcpu_count" {
   type = string
   default = "6"
@@ -187,17 +182,11 @@ build {
     ]
     execute_command = "chmod +x {{ .Path }}; source $HOME/.bash_profile; {{ .Vars }} {{ .Path }}"
   }
-  provisioner "shell" {
-    inline = [
-      "mkdir -p ~/.fastlane/spaceship/${var.xcode_install_user}",
-      "echo ${var.xcversion_auth_cookie} | base64 --decode > ~/.fastlane/spaceship/${var.xcode_install_user}/cookie"
-    ]
-  }
   provisioner "shell" {
     script = "./provision/core/xcode.ps1"
     environment_vars = [
-      "XCODE_INSTALL_USER=${var.xcode_install_user}",
+      "XCODE_INSTALL_STORAGE_URL=${var.xcode_install_storage_url}",
-      "XCODE_INSTALL_PASSWORD=${var.xcode_install_password}"
+      "XCODE_INSTALL_SAS=${var.xcode_install_sas}"
     ]
     execute_command = "chmod +x {{ .Path }}; source $HOME/.bash_profile; {{ .Vars }} pwsh -f {{ .Path }}"
   }

images/macos/templates/macOS-11.json

@@ -12,9 +12,8 @@
         "vm_username": null,
         "vm_password": null,
         "github_api_pat": null,
-        "xcode_install_user": null,
+        "xcode_install_storage_url": null,
-        "xcode_install_password": null,
+        "xcode_install_sas": null,
-        "xcversion_auth_cookie": null,
         "image_os": "macos11"
     },
     "builders": [
@@ -165,22 +164,13 @@
                 "API_PAT={{user `github_api_pat`}}"
             ]
         },
-        {
-            "type": "shell",
-            "inline": "mkdir -p ~/.fastlane/spaceship/{{user `xcode_install_user`}}"
-        },
-        {
-            "type": "file",
-            "source": "{{user `xcversion_auth_cookie`}}",
-            "destination": "~/.fastlane/spaceship/{{user `xcode_install_user`}}/cookie"
-        },
         {
             "type": "shell",
             "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} pwsh -f {{ .Path }}",
             "script": "./provision/core/xcode.ps1",
             "environment_vars": [
-                "XCODE_INSTALL_USER={{user `xcode_install_user`}}",
+                "XCODE_INSTALL_STORAGE_URL={{user `xcode_install_storage_url`}}",
-                "XCODE_INSTALL_PASSWORD={{user `xcode_install_password`}}"
+                "XCODE_INSTALL_SAS={{user `xcode_install_sas`}}"
             ]
         },
         {

images/macos/templates/macOS-12.anka.pkr.hcl

@@ -34,21 +34,16 @@ variable "github_api_pat" {
   default = ""
 }
 
-variable "xcode_install_user" {
+variable "xcode_install_storage_url" {
   type = string
   sensitive = true
 }
 
-variable "xcode_install_password" {
+variable "xcode_install_sas" {
   type = string
   sensitive = true
 }
 
-variable "xcversion_auth_cookie" {
-  type = string
-  default = ""
-}
-
 variable "vcpu_count" {
   type = string
   default = "6"
@@ -188,17 +183,11 @@ build {
     ]
     execute_command = "chmod +x {{ .Path }}; source $HOME/.bash_profile; {{ .Vars }} {{ .Path }}"
   }
-  provisioner "shell" {
-    inline = [
-      "mkdir -p ~/.fastlane/spaceship/${var.xcode_install_user}",
-      "echo ${var.xcversion_auth_cookie} | base64 --decode > ~/.fastlane/spaceship/${var.xcode_install_user}/cookie"
-    ]
-  }
   provisioner "shell" {
     script = "./provision/core/xcode.ps1"
     environment_vars = [
-      "XCODE_INSTALL_USER=${var.xcode_install_user}",
+      "XCODE_INSTALL_STORAGE_URL=${var.xcode_install_storage_url}",
-      "XCODE_INSTALL_PASSWORD=${var.xcode_install_password}"
+      "XCODE_INSTALL_SAS=${var.xcode_install_sas}"
     ]
     execute_command = "chmod +x {{ .Path }}; source $HOME/.bash_profile; {{ .Vars }} pwsh -f {{ .Path }}"
   }

images/macos/templates/macOS-12.json

@@ -12,9 +12,8 @@
         "vm_username": null,
         "vm_password": null,
         "github_api_pat": null,
-        "xcode_install_user": null,
+        "xcode_install_storage_url": null,
-        "xcode_install_password": null,
+        "xcode_install_sas": null,
-        "xcversion_auth_cookie": null,
         "image_os": "macos12"
     },
     "builders": [
@@ -167,22 +166,13 @@
                 "USER_PASSWORD={{user `vm_password`}}"
             ]
         },
-        {
-            "type": "shell",
-            "inline": "mkdir -p ~/.fastlane/spaceship/{{user `xcode_install_user`}}"
-        },
-        {
-            "type": "file",
-            "source": "{{user `xcversion_auth_cookie`}}",
-            "destination": "~/.fastlane/spaceship/{{user `xcode_install_user`}}/cookie"
-        },
         {
             "type": "shell",
             "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} pwsh -f {{ .Path }}",
             "script": "./provision/core/xcode.ps1",
             "environment_vars": [
-                "XCODE_INSTALL_USER={{user `xcode_install_user`}}",
+                "XCODE_INSTALL_STORAGE_URL={{user `xcode_install_storage_url`}}",
-                "XCODE_INSTALL_PASSWORD={{user `xcode_install_password`}}"
+                "XCODE_INSTALL_SAS={{user `xcode_install_sas`}}"
             ]
         },
         {

images/macos/templates/macOS-13.anka.pkr.hcl

@@ -34,21 +34,16 @@ variable "github_api_pat" {
   default = ""
 }
 
-variable "xcode_install_user" {
+variable "xcode_install_storage_url" {
   type = string
   sensitive = true
 }
 
-variable "xcode_install_password" {
+variable "xcode_install_sas" {
   type = string
   sensitive = true
 }
 
-variable "xcversion_auth_cookie" {
-  type = string
-  default = ""
-}
-
 variable "vcpu_count" {
   type = string
   default = "6"
@@ -182,17 +177,11 @@ build {
     ]
     execute_command = "chmod +x {{ .Path }}; source $HOME/.bash_profile; {{ .Vars }} {{ .Path }}"
   }
-  provisioner "shell" {
-    inline = [
-      "mkdir -p ~/.fastlane/spaceship/${var.xcode_install_user}",
-      "echo ${var.xcversion_auth_cookie} | base64 --decode > ~/.fastlane/spaceship/${var.xcode_install_user}/cookie"
-    ]
-  }
   provisioner "shell" {
     script = "./provision/core/xcode.ps1"
     environment_vars = [
-      "XCODE_INSTALL_USER=${var.xcode_install_user}",
+      "XCODE_INSTALL_STORAGE_URL=${var.xcode_install_storage_url}",
-      "XCODE_INSTALL_PASSWORD=${var.xcode_install_password}"
+      "XCODE_INSTALL_SAS=${var.xcode_install_sas}"
     ]
     execute_command = "chmod +x {{ .Path }}; source $HOME/.bash_profile; {{ .Vars }} pwsh -f {{ .Path }}"
   }

images/macos/templates/macOS-13.arm64.anka.pkr.hcl

@@ -34,21 +34,16 @@ variable "github_api_pat" {
   default = ""
 }
 
-variable "xcode_install_user" {
+variable "xcode_install_storage_url" {
   type = string
   sensitive = true
 }
 
-variable "xcode_install_password" {
+variable "xcode_install_sas" {
   type = string
   sensitive = true
 }
 
-variable "xcversion_auth_cookie" {
-  type = string
-  default = ""
-}
-
 variable "vcpu_count" {
   type = string
   default = "6"
@@ -182,17 +177,11 @@ build {
     ]
     execute_command = "chmod +x {{ .Path }}; source $HOME/.bash_profile; {{ .Vars }} {{ .Path }}"
   }
-  provisioner "shell" {
-    inline = [
-      "mkdir -p ~/.fastlane/spaceship/${var.xcode_install_user}",
-      "echo ${var.xcversion_auth_cookie} | base64 --decode > ~/.fastlane/spaceship/${var.xcode_install_user}/cookie"
-    ]
-  }
   provisioner "shell" {
     script = "./provision/core/xcode.ps1"
     environment_vars = [
-      "XCODE_INSTALL_USER=${var.xcode_install_user}",
+      "XCODE_INSTALL_STORAGE_URL=${var.xcode_install_storage_url}",
-      "XCODE_INSTALL_PASSWORD=${var.xcode_install_password}"
+      "XCODE_INSTALL_SAS=${var.xcode_install_sas}"
     ]
     execute_command = "chmod +x {{ .Path }}; source $HOME/.bash_profile; {{ .Vars }} pwsh -f {{ .Path }}"
   }

images/macos/toolsets/toolset-11.json

@@ -3,12 +3,12 @@
         "default": "13.2.1",
         "x64": {
             "versions": [
-                { "link": "13.2.1", "version": "13.2.1", "symlinks": ["13.2"] },
+                { "link": "13.2.1", "version": "13.2.1+13C100", "symlinks": ["13.2"] },
-                { "link": "13.1", "version": "13.1.0" },
+                { "link": "13.1", "version": "13.1.0+13A1030d" },
-                { "link": "13.0", "version": "13.0.0" },
+                { "link": "13.0", "version": "13.0.0+13A233" },
-                { "link": "12.5.1", "version": "12.5.1", "symlinks": ["12.5"] },
+                { "link": "12.5.1", "version": "12.5.1+12E507", "symlinks": ["12.5"] },
-                { "link": "12.4", "version": "12.4.0" },
+                { "link": "12.4", "version": "12.4.0+12D4e" },
-                { "link": "11.7", "version": "11.7.0", "symlinks": ["11.7_beta"] }
+                { "link": "11.7", "version": "11.7.0-GM+11E801a", "symlinks": ["11.7_beta"] }
             ]
         }
     },

images/macos/toolsets/toolset-12.json

@@ -3,13 +3,13 @@
         "default": "14.2",
         "x64": {
             "versions": [
-                { "link": "14.2", "version": "14.2.0" },
+                { "link": "14.2", "version": "14.2.0+14C18" },
-                { "link": "14.1", "version": "14.1.0" },
+                { "link": "14.1", "version": "14.1.0+14B47b" },
-                { "link": "14.0.1", "version": "14.0.1", "symlinks": ["14.0"] },
+                { "link": "14.0.1", "version": "14.0.1+14A400", "symlinks": ["14.0"] },
-                { "link": "13.4.1", "version": "13.4.1", "symlinks": ["13.4"] },
+                { "link": "13.4.1", "version": "13.4.1+13F100", "symlinks": ["13.4"] },
-                { "link": "13.3.1", "version": "13.3.1", "symlinks": ["13.3"] },
+                { "link": "13.3.1", "version": "13.3.1+13E500a", "symlinks": ["13.3"] },
-                { "link": "13.2.1", "version": "13.2.1", "symlinks": ["13.2"] },
+                { "link": "13.2.1", "version": "13.2.1+13C100", "symlinks": ["13.2"] },
-                { "link": "13.1", "version": "13.1.0" }
+                { "link": "13.1", "version": "13.1.0+13A1030d" }
             ]
         }
     },

images/macos/toolsets/toolset-13.json

@@ -3,20 +3,20 @@
         "default": "14.2",
         "x64": {
             "versions": [
-                { "link": "15.0", "version": "15.0.0" },
+                { "link": "15.0", "version": "15.0.0-Beta.2+15A5161b" },
-                { "link": "14.3.1", "version": "14.3.1" },
+                { "link": "14.3.1", "version": "14.3.1+14E300c" },
-                { "link": "14.3", "version": "14.3.0" },
+                { "link": "14.3", "version": "14.3.0+14E222b" },
-                { "link": "14.2", "version": "14.2.0" },
+                { "link": "14.2", "version": "14.2.0+14C18" },
-                { "link": "14.1", "version": "14.1.0" }
+                { "link": "14.1", "version": "14.1.0+14B47b" }
                 ]
         },
         "arm64":{
             "versions": [
-                { "link": "15.0", "version": "15.0.0" },
+                { "link": "15.0", "version": "15.0.0-Beta.2+15A5161b" },
-                { "link": "14.3.1", "version": "14.3.1" },
+                { "link": "14.3.1", "version": "14.3.1+14E300c" },
-                { "link": "14.3", "version": "14.3.0" },
+                { "link": "14.3", "version": "14.3.0+14E222b" },
-                { "link": "14.2", "version": "14.2.0" },
+                { "link": "14.2", "version": "14.2.0+14C18" },
-                { "link": "14.1", "version": "14.1.0" }
+                { "link": "14.1", "version": "14.1.0+14B47b" }
             ]
         }
     },