Switch to build managed image instead of VHD (#8167) (#8208)

2026-01-06 18:19:54 +08:00 · 2023-08-31 16:21:15 +02:00
parent d09a712b71
commit 8077d7b42b
14 changed files with 350 additions and 169 deletions
--- a/images.CI/linux-and-win/azure-pipelines/image-generation.yml
+++ b/images.CI/linux-and-win/azure-pipelines/image-generation.yml
@@ -57,7 +57,7 @@ jobs:
                 -RepoBranch $(CUSTOM_REPOSITORY_BRANCH)

  - task: PowerShell@2
-    displayName: 'Set image template variables'
+    displayName: 'Set variables'
    inputs:
      targetType: 'inline'
      script: |
@@ -71,28 +71,57 @@ jobs:
        Write-Host "##vso[task.setvariable variable=TemplateDirectoryPath;]$TemplateDirectoryPath"
        Write-Host "##vso[task.setvariable variable=TemplatePath;]$TemplatePath"

+        $ManagedImageName = "${{ parameters.image_type }}-$(Build.BuildId)"
+        Write-Host "##vso[task.setvariable variable=ManagedImageName;]$ManagedImageName"
+
+        $VhdName = "$ManagedImageName.vhd"
+        Write-Host "##vso[task.setvariable variable=VhdName;]$VhdName"
+
+        $TempResourceGroupName = "packer-temp-$ManagedImageName"
+        Write-Host "##vso[task.setvariable variable=TempResourceGroupName;]$TempResourceGroupName"
+
  - task: PowerShell@2
    displayName: 'Build VM'
    inputs:
      targetType: filePath
      filePath: ./images.CI/linux-and-win/build-image.ps1
-      arguments: -ResourcesNamePrefix $(Build.BuildId) `
-                        -ClientId $(CLIENT_ID) `
-                        -ClientSecret $(CLIENT_SECRET) `
-                        -TemplatePath $(TemplatePath) `
-                        -ResourceGroup $(AZURE_RESOURCE_GROUP) `
-                        -StorageAccount $(AZURE_STORAGE_ACCOUNT) `
-                        -SubscriptionId $(AZURE_SUBSCRIPTION) `
-                        -TenantId $(AZURE_TENANT) `
-                        -Location $(AZURE_LOCATION) `
-                        -VirtualNetworkName $(BUILD_AGENT_VNET_NAME) `
-                        -VirtualNetworkRG $(BUILD_AGENT_VNET_RESOURCE_GROUP) `
-                        -VirtualNetworkSubnet $(BUILD_AGENT_SUBNET_NAME)
+      arguments: -ClientId $(CLIENT_ID) `
+                 -ClientSecret $(CLIENT_SECRET) `
+                 -TemplatePath $(TemplatePath) `
+                 -ImageName "$(ManagedImageName)" `
+                 -ImageResourceGroupName $(AZURE_RESOURCE_GROUP) `
+                 -TempResourceGroupName "$(TempResourceGroupName)" `
+                 -SubscriptionId $(AZURE_SUBSCRIPTION) `
+                 -TenantId $(AZURE_TENANT) `
+                 -Location $(AZURE_LOCATION) `
+                 -VirtualNetworkName $(BUILD_AGENT_VNET_NAME) `
+                 -VirtualNetworkRG $(BUILD_AGENT_VNET_RESOURCE_GROUP) `
+                 -VirtualNetworkSubnet $(BUILD_AGENT_SUBNET_NAME)

    env:
      PACKER_LOG: 1
      PACKER_LOG_PATH: "$(Agent.TempDirectory)/packer-log.txt"
-  
+
+  - task: PowerShell@2
+    displayName: 'Convert managed image to VHD'
+    inputs:
+      targetType: filePath
+      filePath: ./images.CI/linux-and-win/convert-to-vhd.ps1
+      arguments: -SubscriptionId $(AZURE_SUBSCRIPTION) `
+                 -Location $(AZURE_LOCATION) `
+                 -ResourceGroupName $(AZURE_RESOURCE_GROUP) `
+                 -ManagedImageName "$(ManagedImageName)" `
+                 -GalleryName "github_imagegeneration_convert_to_vhd" `
+                 -GalleryImageSku "${{ parameters.image_type }}" `
+                 -GalleryImageVersion "0.0.$(Build.BuildId)" `
+                 -StorageAccountName $(AZURE_STORAGE_ACCOUNT) `
+                 -StorageAccountContainerName "images" `
+                 -VhdName "$(VhdName)" `
+                 -ClientId $(CLIENT_ID) `
+                 -ClientSecret $(CLIENT_SECRET) `
+                 -TenantId $(AZURE_TENANT) `
+                 -RemoveManagedImage
+
  - task: PowerShell@2
    displayName: 'Copy image artifacts to the separate directory'
    inputs:
@@ -144,11 +173,13 @@ jobs:
        targetType: filePath
        filePath: ./images.CI/linux-and-win/create-release.ps1
        arguments: -BuildId $(Build.BuildId) `
-                          -Organization $(RELEASE_TARGET_ORGANIZATION) `
-                          -DefinitionId $(RELEASE_TARGET_DEFINITION_ID) `
-                          -Project $(RELEASE_TARGET_PROJECT) `
-                          -ImageName ${{ parameters.image_type }} `
-                          -AccessToken $(RELEASE_TARGET_TOKEN)
+                   -Organization $(RELEASE_TARGET_ORGANIZATION) `
+                   -DefinitionId $(RELEASE_TARGET_DEFINITION_ID) `
+                   -Project $(RELEASE_TARGET_PROJECT) `
+                   -ImageName "${{ parameters.image_type }}" `
+                   -StorageAccountContainerName "images" `
+                   -VhdName "$(VhdName)" `
+                   -AccessToken $(RELEASE_TARGET_TOKEN)

  - task: PowerShell@2
    displayName: 'Clean up resources'
@@ -156,10 +187,9 @@ jobs:
    inputs:
      targetType: filePath
      filePath: ./images.CI/linux-and-win/cleanup.ps1
-      arguments: -ResourcesNamePrefix $(Build.BuildId) `
-                     -Image ${{ parameters.image_type }} `
-                     -StorageAccount $(AZURE_STORAGE_ACCOUNT) `
-                     -SubscriptionId $(AZURE_SUBSCRIPTION) `
-                     -ClientId $(CLIENT_ID) `
-                     -ClientSecret $(CLIENT_SECRET) `
-                     -TenantId $(AZURE_TENANT)
+      arguments: -TempResourceGroupName "$(TempResourceGroupName)" `
+                 -StorageAccount $(AZURE_STORAGE_ACCOUNT) `
+                 -SubscriptionId $(AZURE_SUBSCRIPTION) `
+                 -ClientId $(CLIENT_ID) `
+                 -ClientSecret $(CLIENT_SECRET) `
+                 -TenantId $(AZURE_TENANT)
--- a/images.CI/linux-and-win/build-image.ps1
+++ b/images.CI/linux-and-win/build-image.ps1
@@ -2,10 +2,10 @@ param(
    [String] [Parameter (Mandatory=$true)] $TemplatePath,
    [String] [Parameter (Mandatory=$true)] $ClientId,
    [String] [Parameter (Mandatory=$true)] $ClientSecret,
-    [String] [Parameter (Mandatory=$true)] $ResourcesNamePrefix,
    [String] [Parameter (Mandatory=$true)] $Location,
-    [String] [Parameter (Mandatory=$true)] $ResourceGroup,
-    [String] [Parameter (Mandatory=$true)] $StorageAccount,
+    [String] [Parameter (Mandatory=$true)] $ImageName,
+    [String] [Parameter (Mandatory=$true)] $ImageResourceGroupName,
+    [String] [Parameter (Mandatory=$true)] $TempResourceGroupName,
    [String] [Parameter (Mandatory=$true)] $SubscriptionId,
    [String] [Parameter (Mandatory=$true)] $TenantId,
    [String] [Parameter (Mandatory=$false)] $VirtualNetworkName,
@@ -19,8 +19,7 @@ if (-not (Test-Path $TemplatePath))
    exit 1
 }

-$Image = [io.path]::GetFileName($TemplatePath).Split(".")[0]
-$TempResourceGroupName = "${ResourcesNamePrefix}_${Image}"
+$ImageTemplateName = [io.path]::GetFileName($TemplatePath).Split(".")[0]
 $InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper()

 packer validate -syntax-only $TemplatePath
@@ -38,14 +37,13 @@ $SensitiveData = @(
 Write-Host "Show Packer Version"
 packer --version

-Write-Host "Build $Image VM"
-packer build    -var "capture_name_prefix=$ResourcesNamePrefix" `
-                -var "client_id=$ClientId" `
+Write-Host "Build $ImageTemplateName VM"
+packer build    -var "client_id=$ClientId" `
                -var "client_secret=$ClientSecret" `
                -var "install_password=$InstallPassword" `
                -var "location=$Location" `
-                -var "resource_group=$ResourceGroup" `
-                -var "storage_account=$StorageAccount" `
+                -var "managed_image_name=$ImageName" `
+                -var "managed_image_resource_group_name=$ImageResourceGroupName" `
                -var "subscription_id=$SubscriptionId" `
                -var "temp_resource_group_name=$TempResourceGroupName" `
                -var "tenant_id=$TenantId" `
--- a/images.CI/linux-and-win/cleanup.ps1
+++ b/images.CI/linux-and-win/cleanup.ps1
@@ -1,17 +1,14 @@
 param(
-    [String] [Parameter (Mandatory=$true)] $Image,
-    [String] [Parameter (Mandatory=$true)] $ResourcesNamePrefix,
+    [String] [Parameter (Mandatory=$true)] $TempResourceGroupName,
    [String] [Parameter (Mandatory=$true)] $StorageAccount,
+    [String] [Parameter (Mandatory=$true)] $SubscriptionId,
    [String] [Parameter (Mandatory=$true)] $ClientId,
    [String] [Parameter (Mandatory=$true)] $ClientSecret,
-    [String] [Parameter (Mandatory=$true)] $SubscriptionId,
    [String] [Parameter (Mandatory=$true)] $TenantId
 )

 az login --service-principal --username $ClientId --password $ClientSecret --tenant $TenantId | Out-Null

-$TempResourceGroupName = "${ResourcesNamePrefix}_${Image}"
-
 $groupExist = az group exists --name $TempResourceGroupName --subscription $SubscriptionId
 if ($groupExist -eq "true") {
    $osDiskName = az deployment group list --resource-group $TempResourceGroupName --query "[].properties.parameters.osDiskName.value" -o tsv
--- a/images.CI/linux-and-win/convert-to-vhd.ps1
+++ b/images.CI/linux-and-win/convert-to-vhd.ps1
@@ -0,0 +1,185 @@
+param (
+  [String] [Parameter (Mandatory = $true)] $SubscriptionId,
+  [String] [Parameter (Mandatory = $true)] $Location,
+  [String] [Parameter (Mandatory = $true)] $ResourceGroupName,
+  
+  [String] [Parameter (Mandatory = $true)] $ManagedImageName,
+  
+  [String] [Parameter (Mandatory = $true)] $GalleryName,
+  [String] [Parameter (Mandatory = $true)] $GalleryImageSku,
+  [String] [Parameter (Mandatory = $true)] $GalleryImageVersion,
+  
+  [String] [Parameter (Mandatory = $true)] $StorageAccountName,
+  [String] [Parameter (Mandatory = $true)] $StorageAccountContainerName,
+  [String] [Parameter (Mandatory = $true)] $VhdName,
+
+  [Switch] [Parameter (Mandatory = $false)] $RemoveManagedImage,
+  
+  [String] [Parameter (Mandatory = $true)] $ClientId,
+  [String] [Parameter (Mandatory = $true)] $ClientSecret,
+  [String] [Parameter (Mandatory = $true)] $TenantId
+)
+
+$ErrorActionPreference = "Stop"
+
+# Login to Azure
+az login --service-principal --username $ClientId --password $ClientSecret --tenant $TenantId | Out-Null
+az account set --subscription $SubscriptionId | Out-Null
+
+# Create Compute Image Gallery if it doesn't exist
+Write-Host "Creating Compute Image Gallery '$GalleryName'..."
+$galleryExists = az sig list --resource-group $ResourceGroupName --query "[?name=='$GalleryName']" -o tsv
+if ($null -eq $galleryExists) {
+  az sig create --resource-group $ResourceGroupName --gallery-name $GalleryName --location $Location
+}
+
+# Create Image Definition if it doesn't exist
+Write-Host "Creating Image Definition '$ManagedImageName'..."
+$imageDefinitionName = "RunnerImage-$GalleryImageSku"
+if ($ManagedImageName -like "*windows*") {
+  $imageOsType = "Windows"
+} elseif ($ManagedImageName -like "*ubuntu*") {
+  $imageOsType = "Linux"
+} else {
+  throw "Unknown OS type for image '$ManagedImageName'"
+}
+
+$galleryImageExists = az sig image-definition list --resource-group $ResourceGroupName --gallery-name $GalleryName --query "[?name=='$imageDefinitionName']" -o tsv
+if ($null -eq $galleryImageExists) {
+  az sig image-definition create `
+    --resource-group $ResourceGroupName `
+    --gallery-name $GalleryName `
+    --gallery-image-definition $imageDefinitionName `
+    --publisher "GuthubImagesGeneration" `
+    --offer "RunnerImage" `
+    --sku $GalleryImageSku `
+    --os-type $imageOsType `
+    --location $Location
+}
+
+# Create Image Version from existing Managed Image
+Write-Host "Creating Image Version '$GalleryImageVersion' from Managed Image '$ManagedImageName'..."
+az sig image-version create `
+  --resource-group $ResourceGroupName `
+  --gallery-name $GalleryName `
+  --gallery-image-definition $imageDefinitionName `
+  --gallery-image-version $GalleryImageVersion `
+  --managed-image "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.Compute/images/$ManagedImageName" `
+  --target-regions $Location `
+  --replica-count 1 `
+  --location $Location
+
+# Create Azure Managed Disk from Shared Image Gallery
+Write-Host "Creating Azure Managed Disk '$ManagedImageName' from Shared Image Gallery..."
+az disk create `
+  --resource-group $ResourceGroupName `
+  --name $ManagedImageName `
+  --location $Location `
+  --gallery-image-reference "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.Compute/galleries/$GalleryName/images/$imageDefinitionName/versions/$GalleryImageVersion"
+
+# Generate SAS URL for the Managed Disk
+Write-Host "Generating SAS URL for the Managed Disk '$ManagedImageName'..."
+$sourceDiskUri = az disk grant-access `
+  --resource-group $ResourceGroupName `
+  --name $ManagedImageName `
+  --duration-in-seconds 86400 `
+  --access-level Read `
+  --query [accessSas] -o tsv
+
+Write-Host "Querying key for the storage account '$StorageAccountName'..."
+$targetKey = az storage account keys list `
+  --resource-group $ResourceGroupName `
+  --account-name $StorageAccountName `
+  --query "[0].value" -o tsv
+
+Write-Host ("Copying VHD blob from '{0}' to 'https://{1}.blob.core.windows.net/{2}/{3}'..." `
+    -f $sourceDiskUri.Split('?')[0], $StorageAccountName, $StorageAccountContainerName, $VhdName)
+
+az storage blob copy start `
+  --source-uri """$sourceDiskUri""" `
+  --destination-blob $VhdName `
+  --destination-container $StorageAccountContainerName `
+  --account-name $StorageAccountName `
+  --account-key $targetKey `
+  --only-show-errors
+
+Write-Host "Waiting for the copy to complete..."
+while ($true) {
+  $status = az storage blob show `
+    --container-name $StorageAccountContainerName `
+    --name $VhdName `
+    --account-name $StorageAccountName `
+    --account-key $targetKey `
+    --query "properties.copy.status" -o tsv
+
+  if ($status -eq "success") {
+    Write-Host "Copy completed successfully."
+    break
+  } elseif ($status -ne "pending") {
+    Write-Host "Copy failed with status '$status', see blob information below:"
+    az storage blob show `
+      --container-name $StorageAccountContainerName `
+      --name $VhdName `
+      --account-name $StorageAccountName `
+      --account-key $targetKey
+    throw "Copy failed with status '$status'"
+  }
+
+  $progress = az storage blob show `
+    --container-name $StorageAccountContainerName `
+    --name $VhdName `
+    --account-name $StorageAccountName `
+    --account-key $targetKey `
+    --query "properties.copy.progress" -o tsv
+
+  Write-Host "Progress: $(($progress.Split("/")[0] / $progress.Split("/")[1]).ToString("P"))"
+  Start-Sleep -Seconds 15
+}
+
+Write-Host "Successfully converted '$ManagedImageName' to '$VhdName' in '$StorageAccountName' storage account."
+
+# Remove Managed Image if requested
+if ($RemoveManagedImage) {
+  Write-Host "Removing Managed Image '$ManagedImageName'..."
+  az image delete `
+    --resource-group $ResourceGroupName `
+    --name $ManagedImageName `
+    --only-show-errors
+  if ($LastExitCode) {
+    Write-Host "Warning: Failed to delete the Managed Image '$ManagedImageName'."
+  }
+}
+
+Write-Host "Cleaning up..."
+
+# Revoke SAS URL for the Managed Disk
+az disk revoke-access `
+  --resource-group $ResourceGroupName `
+  --name $ManagedImageName `
+  --only-show-errors
+if ($LastExitCode) {
+  Write-Host "Warning: Failed to revoke access to the Managed Disk '$ManagedImageName'."
+}
+
+# Delete Azure Managed Disk from Shared Image Gallery
+az disk delete `
+  --resource-group $ResourceGroupName `
+  --name $ManagedImageName `
+  --only-show-errors `
+  --yes
+if ($LastExitCode) {
+  Write-Host "Warning: Failed to delete the Managed Disk '$ManagedImageName'."
+}
+
+# Delete Image Version from Shared Image Gallery
+az sig image-version delete `
+  --resource-group $ResourceGroupName `
+  --gallery-name $GalleryName `
+  --gallery-image-definition $imageDefinitionName `
+  --gallery-image-version $GalleryImageVersion `
+  --only-show-errors
+if ($LastExitCode) {
+  Write-Host "Warning: Failed to delete the Image Version '$GalleryImageVersion'."
+}
+
+Write-Host "Done."
--- a/images.CI/linux-and-win/create-release.ps1
+++ b/images.CI/linux-and-win/create-release.ps1
@@ -3,6 +3,8 @@ param(
    [String] [Parameter (Mandatory)] $Organization,
    [String] [Parameter (Mandatory)] $Project,
    [String] [Parameter (Mandatory)] $ImageName,
+    [String] [Parameter (Mandatory)] $StorageAccountContainerName,
+    [String] [Parameter (Mandatory)] $VhdName,
    [String] [Parameter (Mandatory)] $DefinitionId,
    [String] [Parameter (Mandatory)] $AccessToken
 )
@@ -16,6 +18,12 @@ $Body = @{
      ImageName = @{
        value = $ImageName
      }
+      ImageStorageContainerName = @{
+        value = $StorageAccountContainerName
+      }
+      ImageBlobPath = @{
+        value = $VhdName
+      }
    }
    isDraft = "false"
 } | ConvertTo-Json -Depth 3