From 9b52e4043f96d7d6871cc7903271e328207f3c58 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Fri, 14 Feb 2020 19:09:09 +0300 Subject: [PATCH 01/22] Configure yml files for the images --- .../azure-pipelines/image-generation.yml | 29 +++++++++++++ images.CI/azure-pipelines/ubuntu1604.yml | 22 ++++++++++ images.CI/azure-pipelines/ubuntu1804.yml | 22 ++++++++++ images.CI/azure-pipelines/windows2016.yml | 22 ++++++++++ images.CI/azure-pipelines/windows2019.yml | 22 ++++++++++ images.CI/build-image.ps1 | 41 +++++++++++++++++++ images.CI/cleanup.ps1 | 25 +++++++++++ 7 files changed, 183 insertions(+) create mode 100644 images.CI/azure-pipelines/image-generation.yml create mode 100644 images.CI/azure-pipelines/ubuntu1604.yml create mode 100644 images.CI/azure-pipelines/ubuntu1804.yml create mode 100644 images.CI/azure-pipelines/windows2016.yml create mode 100644 images.CI/azure-pipelines/windows2019.yml create mode 100644 images.CI/build-image.ps1 create mode 100644 images.CI/cleanup.ps1 diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml new file mode 100644 index 00000000..373c7500 --- /dev/null +++ b/images.CI/azure-pipelines/image-generation.yml @@ -0,0 +1,29 @@ +jobs: +- job: + pool: ci-agent-pool + steps: + - script: | + ./images.CI/build-image.ps1 -ResourcesNamePrefix $(Build.BuildNumber) ` + -ClientId $(CLIENT_ID) ` + -ClientSecret $(CLIENT_SECRET) ` + -Image ${{ parameters.image_type }} ` + -ResourceGroup $(AZURE_RESOURCE_GROUP) ` + -StorageAccount $(AZURE_STORAGE_ACCOUNT) ` + -SubscriptionId $(AZURE_SUBSCRIPTION) ` + -TenantId $(AZURE_TENANT) ` + -Location $(AZURE_LOCATION) ` + -VirtualNetworkName $(BUILD_AGENT_VNET_NAME) ` + -VirtualNetworkRG $(BUILD_AGENT_VNET_RESOURCE_GROUP) ` + -VirtualNetworkSubnet $(BUILD_AGENT_SUBNET_NAME) ` + -GitHubFeedToken $(GITHUB_TOKEN) + displayName: Build VM + + - script: | + ./images.CI/cleanup.ps1 -ResourcesNamePrefix $(Build.BuildNumber) ` + -ClientId $(CLIENT_ID) ` + -ClientSecret $(CLIENT_SECRET) ` + -Image ${{ parameters.image_type }} ` + -SubscriptionId $(AZURE_SUBSCRIPTION) ` + -TenantId $(AZURE_TENANT) + displayName: Clean up resources + condition: always() \ No newline at end of file diff --git a/images.CI/azure-pipelines/ubuntu1604.yml b/images.CI/azure-pipelines/ubuntu1604.yml new file mode 100644 index 00000000..c76a3924 --- /dev/null +++ b/images.CI/azure-pipelines/ubuntu1604.yml @@ -0,0 +1,22 @@ +# schedules: +# - cron: "0 0 * * *" +# displayName: Daily +# branches: +# include: +# - master +# always: true + +variables: +- group: "Image Generation Variables" + +trigger: none +pr: + autoCancel: true + branches: + include: + - master + +jobs: +- template: image-generation.yml + parameters: + image_type: ubuntu1604 \ No newline at end of file diff --git a/images.CI/azure-pipelines/ubuntu1804.yml b/images.CI/azure-pipelines/ubuntu1804.yml new file mode 100644 index 00000000..eb7c6dd0 --- /dev/null +++ b/images.CI/azure-pipelines/ubuntu1804.yml @@ -0,0 +1,22 @@ +# schedules: +# - cron: "0 0 * * *" +# displayName: Daily +# branches: +# include: +# - master +# always: true + +variables: +- group: "Image Generation Variables" + +trigger: none +pr: + autoCancel: true + branches: + include: + - master + +jobs: +- template: image-generation.yml + parameters: + image_type: ubuntu1804 \ No newline at end of file diff --git a/images.CI/azure-pipelines/windows2016.yml b/images.CI/azure-pipelines/windows2016.yml new file mode 100644 index 00000000..d3ac23e6 --- /dev/null +++ b/images.CI/azure-pipelines/windows2016.yml @@ -0,0 +1,22 @@ +# schedules: +# - cron: "0 0 * * *" +# displayName: Daily +# branches: +# include: +# - master +# always: true + +variables: +- group: "Image Generation Variables" + +trigger: none +pr: + autoCancel: true + branches: + include: + - master + +jobs: +- template: image-generation.yml + parameters: + image_type: Windows2016-Azure \ No newline at end of file diff --git a/images.CI/azure-pipelines/windows2019.yml b/images.CI/azure-pipelines/windows2019.yml new file mode 100644 index 00000000..93c73003 --- /dev/null +++ b/images.CI/azure-pipelines/windows2019.yml @@ -0,0 +1,22 @@ +# schedules: +# - cron: "0 0 * * *" +# displayName: Daily +# branches: +# include: +# - master +# always: true + +variables: +- group: "Image Generation Variables" + +trigger: none +pr: + autoCancel: true + branches: + include: + - master + +jobs: +- template: image-generation.yml + parameters: + image_type: Windows2019-Azure \ No newline at end of file diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 new file mode 100644 index 00000000..1087abbe --- /dev/null +++ b/images.CI/build-image.ps1 @@ -0,0 +1,41 @@ +param( + [ValidateSet('Windows2019-Azure','Windows2016-Azure','ubuntu1604','ubuntu1804')] + [String] [Parameter (Mandatory=$true)] $Image, + [String] [Parameter (Mandatory=$true)] $ClientId, + [String] [Parameter (Mandatory=$true)] $ClientSecret, + [String] [Parameter (Mandatory=$true)] $GitHubFeedToken, + [String] [Parameter (Mandatory=$true)] $ResourcesNamePrefix, + [String] [Parameter (Mandatory=$true)] $Location, + [String] [Parameter (Mandatory=$true)] $ResourceGroup, + [String] [Parameter (Mandatory=$true)] $StorageAccount, + [String] [Parameter (Mandatory=$true)] $SubscriptionId, + [String] [Parameter (Mandatory=$true)] $TenantId, + [String] [Parameter (Mandatory=$true)] $VirtualNetworkName, + [String] [Parameter (Mandatory=$true)] $VirtualNetworkRG, + [String] [Parameter (Mandatory=$true)] $VirtualNetworkSubnet +) + +$TemplatePath = (Get-ChildItem -Path "images" -Include "$Image.json" -Recurse -Depth 2).FullName +$TempResourceGroupName = "${ResourcesNamePrefix}_${Image}" +$InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper() + +Write-Host "TemplatePath = $TemplatePath" +Write-Host "ResourcesNamePrefix = $ResourcesNamePrefix" +Write-Host "TempResourceGroupName = $TempResourceGroupName" + +Write-Host "Build $Image VM" +packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` + -var "client_id=$ClientId" ` + -var "client_secret=$ClientSecret" ` + -var "install_password=$InstallPassword" ` + -var "github_feed_token=$GitHubFeedToken" ` + -var "location=$Location" ` + -var "resource_group=$ResourceGroup" ` + -var "storage_account=$StorageAccount" ` + -var "subscription_id=$SubscriptionId" ` + -var "temp_resource_group_name=$TempResourceGroupName" ` + -var "tenant_id=$TenantId" ` + -var "virtual_network_name=$VirtualNetworkName" ` + -var "virtual_network_resource_group_name=$VirtualNetworkRG" ` + -var "virtual_network_subnet_name=$VirtualNetworkSubnet" ` + $TemplatePath \ No newline at end of file diff --git a/images.CI/cleanup.ps1 b/images.CI/cleanup.ps1 new file mode 100644 index 00000000..fcf7d946 --- /dev/null +++ b/images.CI/cleanup.ps1 @@ -0,0 +1,25 @@ +param( + [ValidateSet('Windows2019-Azure','Windows2016-Azure','ubuntu1604','ubuntu1804')] + [String] [Parameter (Mandatory=$true)] $Image, + [String] [Parameter (Mandatory=$true)] $ResourcesNamePrefix, + [String] [Parameter (Mandatory=$true)] $ClientId, + [String] [Parameter (Mandatory=$true)] $ClientSecret, + [String] [Parameter (Mandatory=$true)] $SubscriptionId, + [String] [Parameter (Mandatory=$true)] $TenantId +) + +az login --service-principal --username $ClientId --password $ClientSecret --tenant $TenantId | Out-Null + +$TempResourceGroupName = "${ResourcesNamePrefix}_${Image}" + +Write-Host "ResourcesNamePrefix = $ResourcesNamePrefix" +Write-Host "TempResourceGroupName = $TempResourceGroupName" + +$groupExist = az group exists --name $TempResourceGroupName --subscription $SubscriptionId | Out-Null +if ($groupExist -eq "true") { + Write-Host "Found a match, deleting temporary files" + az group delete --name $TempResourceGroupName --subscription $SubscriptionId --yes | Out-Null + Write-Host "Temporary group was deleted succesfully" -ForegroundColor Green +} else { + Write-Host "No temporary groups found" +} \ No newline at end of file From 78ce3a0bd3b903f540b18829e2c83574ddeeafbb Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Fri, 14 Feb 2020 19:44:06 +0300 Subject: [PATCH 02/22] Change the script steps to the PowerShell tasks --- .../azure-pipelines/image-generation.yml | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 373c7500..98922bf0 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -2,8 +2,12 @@ jobs: - job: pool: ci-agent-pool steps: - - script: | - ./images.CI/build-image.ps1 -ResourcesNamePrefix $(Build.BuildNumber) ` + - task: PowerShell@2 + displayName: 'Build VM' + inputs: + targetType: filePath + filePath: ./images.CI/build-image.ps1 + arguments: -ResourcesNamePrefix $(Build.BuildNumber) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` -Image ${{ parameters.image_type }} ` @@ -16,14 +20,16 @@ jobs: -VirtualNetworkRG $(BUILD_AGENT_VNET_RESOURCE_GROUP) ` -VirtualNetworkSubnet $(BUILD_AGENT_SUBNET_NAME) ` -GitHubFeedToken $(GITHUB_TOKEN) - displayName: Build VM - - script: | - ./images.CI/cleanup.ps1 -ResourcesNamePrefix $(Build.BuildNumber) ` + - task: PowerShell@2 + displayName: 'Clean up resources' + condition: always() + inputs: + targetType: filePath + filePath: ./images.CI/cleanup.ps1 + arguments: -ResourcesNamePrefix $(Build.BuildNumber) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` -Image ${{ parameters.image_type }} ` -SubscriptionId $(AZURE_SUBSCRIPTION) ` - -TenantId $(AZURE_TENANT) - displayName: Clean up resources - condition: always() \ No newline at end of file + -TenantId $(AZURE_TENANT) \ No newline at end of file From 0295b6f57eb5865b228587685b557d19839d7e77 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Sat, 15 Feb 2020 20:52:05 +0300 Subject: [PATCH 03/22] Move variables group --- images.CI/azure-pipelines/image-generation.yml | 4 ++++ images.CI/azure-pipelines/ubuntu1604.yml | 3 --- images.CI/azure-pipelines/ubuntu1804.yml | 3 --- images.CI/azure-pipelines/windows2016.yml | 3 --- images.CI/azure-pipelines/windows2019.yml | 3 --- 5 files changed, 4 insertions(+), 12 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 98922bf0..2f9840b6 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -1,6 +1,10 @@ jobs: - job: pool: ci-agent-pool + timeoutInMinutes: 600 + variables: + - group: Image Generation Variables + steps: - task: PowerShell@2 displayName: 'Build VM' diff --git a/images.CI/azure-pipelines/ubuntu1604.yml b/images.CI/azure-pipelines/ubuntu1604.yml index c76a3924..b19e79a1 100644 --- a/images.CI/azure-pipelines/ubuntu1604.yml +++ b/images.CI/azure-pipelines/ubuntu1604.yml @@ -6,9 +6,6 @@ # - master # always: true -variables: -- group: "Image Generation Variables" - trigger: none pr: autoCancel: true diff --git a/images.CI/azure-pipelines/ubuntu1804.yml b/images.CI/azure-pipelines/ubuntu1804.yml index eb7c6dd0..db8acfd9 100644 --- a/images.CI/azure-pipelines/ubuntu1804.yml +++ b/images.CI/azure-pipelines/ubuntu1804.yml @@ -6,9 +6,6 @@ # - master # always: true -variables: -- group: "Image Generation Variables" - trigger: none pr: autoCancel: true diff --git a/images.CI/azure-pipelines/windows2016.yml b/images.CI/azure-pipelines/windows2016.yml index d3ac23e6..eaac2d1b 100644 --- a/images.CI/azure-pipelines/windows2016.yml +++ b/images.CI/azure-pipelines/windows2016.yml @@ -6,9 +6,6 @@ # - master # always: true -variables: -- group: "Image Generation Variables" - trigger: none pr: autoCancel: true diff --git a/images.CI/azure-pipelines/windows2019.yml b/images.CI/azure-pipelines/windows2019.yml index 93c73003..ebbf68e7 100644 --- a/images.CI/azure-pipelines/windows2019.yml +++ b/images.CI/azure-pipelines/windows2019.yml @@ -6,9 +6,6 @@ # - master # always: true -variables: -- group: "Image Generation Variables" - trigger: none pr: autoCancel: true From 624b905bc9e125cc3f38cbea9f53c06b0f028eab Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 17 Feb 2020 13:48:05 +0300 Subject: [PATCH 04/22] Remove log messages --- images.CI/build-image.ps1 | 4 ---- images.CI/cleanup.ps1 | 3 --- 2 files changed, 7 deletions(-) diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 index 1087abbe..989167e4 100644 --- a/images.CI/build-image.ps1 +++ b/images.CI/build-image.ps1 @@ -19,10 +19,6 @@ $TemplatePath = (Get-ChildItem -Path "images" -Include "$Image.json" -Recurse -D $TempResourceGroupName = "${ResourcesNamePrefix}_${Image}" $InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper() -Write-Host "TemplatePath = $TemplatePath" -Write-Host "ResourcesNamePrefix = $ResourcesNamePrefix" -Write-Host "TempResourceGroupName = $TempResourceGroupName" - Write-Host "Build $Image VM" packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` -var "client_id=$ClientId" ` diff --git a/images.CI/cleanup.ps1 b/images.CI/cleanup.ps1 index fcf7d946..4b0b527b 100644 --- a/images.CI/cleanup.ps1 +++ b/images.CI/cleanup.ps1 @@ -12,9 +12,6 @@ az login --service-principal --username $ClientId --password $ClientSecret --ten $TempResourceGroupName = "${ResourcesNamePrefix}_${Image}" -Write-Host "ResourcesNamePrefix = $ResourcesNamePrefix" -Write-Host "TempResourceGroupName = $TempResourceGroupName" - $groupExist = az group exists --name $TempResourceGroupName --subscription $SubscriptionId | Out-Null if ($groupExist -eq "true") { Write-Host "Found a match, deleting temporary files" From 9b8fe11d2cb0dd83fe3405c445d36bcbb3bd1390 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 17 Feb 2020 17:25:55 +0300 Subject: [PATCH 05/22] Remove ValidateSet --- images.CI/build-image.ps1 | 9 ++++++++- images.CI/cleanup.ps1 | 1 - 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 index 989167e4..9a35373d 100644 --- a/images.CI/build-image.ps1 +++ b/images.CI/build-image.ps1 @@ -1,5 +1,4 @@ param( - [ValidateSet('Windows2019-Azure','Windows2016-Azure','ubuntu1604','ubuntu1804')] [String] [Parameter (Mandatory=$true)] $Image, [String] [Parameter (Mandatory=$true)] $ClientId, [String] [Parameter (Mandatory=$true)] $ClientSecret, @@ -16,9 +15,17 @@ param( ) $TemplatePath = (Get-ChildItem -Path "images" -Include "$Image.json" -Recurse -Depth 2).FullName +if ($TemplatePath -eq $null) +{ + Write-Error "'-Image' parameter is not valid. You have to specify correct image type." + exit 1 +} + $TempResourceGroupName = "${ResourcesNamePrefix}_${Image}" $InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper() +packer validate -syntax-only $TemplatePath + Write-Host "Build $Image VM" packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` -var "client_id=$ClientId" ` diff --git a/images.CI/cleanup.ps1 b/images.CI/cleanup.ps1 index 4b0b527b..00050d55 100644 --- a/images.CI/cleanup.ps1 +++ b/images.CI/cleanup.ps1 @@ -1,5 +1,4 @@ param( - [ValidateSet('Windows2019-Azure','Windows2016-Azure','ubuntu1604','ubuntu1804')] [String] [Parameter (Mandatory=$true)] $Image, [String] [Parameter (Mandatory=$true)] $ResourcesNamePrefix, [String] [Parameter (Mandatory=$true)] $ClientId, From 60dc082e2fc2edc7a70923f258699fc85869b8f2 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 17 Feb 2020 17:31:08 +0300 Subject: [PATCH 06/22] Minor update --- images.CI/build-image.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 index 9a35373d..6b75ec28 100644 --- a/images.CI/build-image.ps1 +++ b/images.CI/build-image.ps1 @@ -15,7 +15,7 @@ param( ) $TemplatePath = (Get-ChildItem -Path "images" -Include "$Image.json" -Recurse -Depth 2).FullName -if ($TemplatePath -eq $null) +if (-not $TemplatePath) { Write-Error "'-Image' parameter is not valid. You have to specify correct image type." exit 1 From 6aeaad42ea055ec3720666015c910750e7d454b0 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 18 Feb 2020 15:37:23 +0300 Subject: [PATCH 07/22] Add step with download custom repository --- images.CI/azure-pipelines/image-generation.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 2f9840b6..459b55c3 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -6,6 +6,13 @@ jobs: - group: Image Generation Variables steps: + - pwsh: | + Remove-Item -path './*' -Recurse -Force + Write-Host "Download $(BRANCH_NAME) branch from $(IMAGE_GEN_REPO_URI)" + git clone $(IMAGE_GEN_REPO_URI) . -b $(BRANCH_NAME) --single-branch --depth 1 + displayName: 'Download custom repository' + condition: and(ne(variables['BRANCH_NAME'], ''), ne(variables['IMAGE_GEN_REPO_URI'], '')) + - task: PowerShell@2 displayName: 'Build VM' inputs: From b1a3c37984d814d4a1ecb39b5cf4932c773727dc Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 18 Feb 2020 16:50:11 +0300 Subject: [PATCH 08/22] Add logs filter --- images.CI/build-image.ps1 | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 index 6b75ec28..dabbb069 100644 --- a/images.CI/build-image.ps1 +++ b/images.CI/build-image.ps1 @@ -26,6 +26,8 @@ $InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper() packer validate -syntax-only $TemplatePath +$SensitiveData = @('OSType', 'StorageAccountLocation', 'OSDiskUri', 'OSDiskUriReadOnlySas', 'TemplateUri', 'TemplateUriReadOnlySas', ': ->') + Write-Host "Build $Image VM" packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` -var "client_id=$ClientId" ` @@ -41,4 +43,10 @@ packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` -var "virtual_network_name=$VirtualNetworkName" ` -var "virtual_network_resource_group_name=$VirtualNetworkRG" ` -var "virtual_network_subnet_name=$VirtualNetworkSubnet" ` - $TemplatePath \ No newline at end of file + $TemplatePath ` + | Where-Object { + #Filter sensitive data from Packer logs + $currentString = $_ + $matchedString = $SensitiveData | Where-Object { $currentString -match $_ } + return $matchedString -eq $null + } \ No newline at end of file From 9c41221b9ba903c48bb94173e0a9022d5d865b54 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 18 Feb 2020 17:05:00 +0300 Subject: [PATCH 09/22] Minor updates --- images.CI/build-image.ps1 | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 index dabbb069..73455c02 100644 --- a/images.CI/build-image.ps1 +++ b/images.CI/build-image.ps1 @@ -26,7 +26,13 @@ $InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper() packer validate -syntax-only $TemplatePath -$SensitiveData = @('OSType', 'StorageAccountLocation', 'OSDiskUri', 'OSDiskUriReadOnlySas', 'TemplateUri', 'TemplateUriReadOnlySas', ': ->') +$SensitiveData = @('OSType', + 'StorageAccountLocation', + 'OSDiskUri', + 'OSDiskUriReadOnlySas', + 'TemplateUri', + 'TemplateUriReadOnlySas', + ': ->') Write-Host "Build $Image VM" packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` @@ -47,6 +53,6 @@ packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` | Where-Object { #Filter sensitive data from Packer logs $currentString = $_ - $matchedString = $SensitiveData | Where-Object { $currentString -match $_ } - return $matchedString -eq $null + $sensitiveString = $SensitiveData | Where-Object { $currentString -match $_ } + $sensitiveString -eq $null } \ No newline at end of file From f4bd0f2a6a7b6d72f6570b5ac3a873c4b8de85b8 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 18 Feb 2020 17:09:48 +0300 Subject: [PATCH 10/22] Minor update --- images.CI/build-image.ps1 | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 index 73455c02..54748e41 100644 --- a/images.CI/build-image.ps1 +++ b/images.CI/build-image.ps1 @@ -26,13 +26,15 @@ $InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper() packer validate -syntax-only $TemplatePath -$SensitiveData = @('OSType', - 'StorageAccountLocation', - 'OSDiskUri', - 'OSDiskUriReadOnlySas', - 'TemplateUri', - 'TemplateUriReadOnlySas', - ': ->') +$SensitiveData = @( + 'OSType', + 'StorageAccountLocation', + 'OSDiskUri', + 'OSDiskUriReadOnlySas', + 'TemplateUri', + 'TemplateUriReadOnlySas', + ': ->' +) Write-Host "Build $Image VM" packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` From 33a765dcc51a12204bbc89906f1217715b838542 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 18 Feb 2020 17:19:11 +0300 Subject: [PATCH 11/22] Change pwsh step to powershell step --- images.CI/azure-pipelines/image-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 459b55c3..bab1c1c8 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -6,7 +6,7 @@ jobs: - group: Image Generation Variables steps: - - pwsh: | + - powershell: | Remove-Item -path './*' -Recurse -Force Write-Host "Download $(BRANCH_NAME) branch from $(IMAGE_GEN_REPO_URI)" git clone $(IMAGE_GEN_REPO_URI) . -b $(BRANCH_NAME) --single-branch --depth 1 From 82bdea2dfe53bb63696e63fe5c6b92a0c87d13b6 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Wed, 19 Feb 2020 10:58:59 +0300 Subject: [PATCH 12/22] Update variables names --- images.CI/azure-pipelines/image-generation.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index bab1c1c8..0c05a7e7 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -8,10 +8,10 @@ jobs: steps: - powershell: | Remove-Item -path './*' -Recurse -Force - Write-Host "Download $(BRANCH_NAME) branch from $(IMAGE_GEN_REPO_URI)" - git clone $(IMAGE_GEN_REPO_URI) . -b $(BRANCH_NAME) --single-branch --depth 1 + Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" + git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 displayName: 'Download custom repository' - condition: and(ne(variables['BRANCH_NAME'], ''), ne(variables['IMAGE_GEN_REPO_URI'], '')) + condition: and(ne(variables['CUSTOM_REPOSITORY_BRANCH'], ''), ne(variables['CUSTOM_REPOSITORY_URL'], '')) - task: PowerShell@2 displayName: 'Build VM' From 68bde6422205d1b7aea26258f3b0376ea7120b01 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Wed, 19 Feb 2020 14:30:44 +0300 Subject: [PATCH 13/22] Change step to PowerShell task --- images.CI/azure-pipelines/image-generation.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 0c05a7e7..477cdc5f 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -6,12 +6,15 @@ jobs: - group: Image Generation Variables steps: - - powershell: | - Remove-Item -path './*' -Recurse -Force - Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" - git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 + - task: PowerShell@2 displayName: 'Download custom repository' condition: and(ne(variables['CUSTOM_REPOSITORY_BRANCH'], ''), ne(variables['CUSTOM_REPOSITORY_URL'], '')) + inputs: + targetType: 'inline' + script: | + Remove-Item -path './*' -Recurse -Force + Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" + git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 - task: PowerShell@2 displayName: 'Build VM' From 56bc08104e91be4946effd0f070337a49f0dfb6f Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Thu, 20 Feb 2020 17:52:12 +0300 Subject: [PATCH 14/22] Add deployment step (#4) * add release step * add creation release * move file to another folder * resolve comments * minor changes --- .../azure-pipelines/image-generation.yml | 12 +++++++ images.CI/create-release.ps1 | 31 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 images.CI/create-release.ps1 diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 477cdc5f..eb0136eb 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -35,6 +35,18 @@ jobs: -VirtualNetworkSubnet $(BUILD_AGENT_SUBNET_NAME) ` -GitHubFeedToken $(GITHUB_TOKEN) + - task: PowerShell@2 + displayName: 'Create release for VM deployment' + inputs: + targetType: filePath + filePath: ./images.CI/create-release.ps1 + arguments: -BuildId $(Build.BuildNumber) ` + -Organization $(RELEASE_TARGET_ORGANIZATION) ` + -DefinitionId $(RELEASE_TARGET_DEFINITION_ID) ` + -Project $(RELEASE_TARGET_PROJECT) ` + -ImageName ${{ parameters.image_type }} ` + -AccessToken $(RELEASE_TARGET_TOKEN) + - task: PowerShell@2 displayName: 'Clean up resources' condition: always() diff --git a/images.CI/create-release.ps1 b/images.CI/create-release.ps1 new file mode 100644 index 00000000..72d0db4e --- /dev/null +++ b/images.CI/create-release.ps1 @@ -0,0 +1,31 @@ +param( + [UInt32] [Parameter (Mandatory)] $BuildId, + [String] [Parameter (Mandatory)] $Organization, + [String] [Parameter (Mandatory)] $Project, + [String] [Parameter (Mandatory)] $ImageName, + [String] [Parameter (Mandatory)] $DefinitionId, + [String] [Parameter (Mandatory)] $AccessToken +) + +$Body = @{ + definitionId = $DefinitionId + variables = @{ + ImageBuildId = @{ + value = $BuildId + } + ImageName = @{ + value = $ImageName + } + } + isDraft = "false" +} | ConvertTo-Json -Depth 3 + +$URL = "https://vsrm.dev.azure.com/$Organization/$Project/_apis/release/releases?api-version=5.1" +$base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("'':${AccessToken}")) +$headers = @{ + Authorization = "Basic ${base64AuthInfo}" +} + +$NewRelease = Invoke-RestMethod $URL -Body $Body -Method "POST" -Headers $headers -ContentType "application/json" + +Write-Host "Created release: $($NewRelease._links.web.href)" \ No newline at end of file From 50cd92aade7a77fdfeed8893ff825d7921866804 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Thu, 20 Feb 2020 19:10:36 +0300 Subject: [PATCH 15/22] Fix github issue --- images.CI/azure-pipelines/image-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index eb0136eb..20adde52 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -14,7 +14,7 @@ jobs: script: | Remove-Item -path './*' -Recurse -Force Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" - git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 + cmd /c "git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 2>&1" - task: PowerShell@2 displayName: 'Build VM' From aa4c025a44514d590048f867353da1a5dc47b47c Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Thu, 20 Feb 2020 22:23:17 +0300 Subject: [PATCH 16/22] Add comment --- images.CI/azure-pipelines/image-generation.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 20adde52..d4fdf0ea 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -14,6 +14,8 @@ jobs: script: | Remove-Item -path './*' -Recurse -Force Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" + # git in powershell sometimes broke the AzDO pipelines with incorrect stdout interpretation + # we redirect the git output to stdout manually to avoid this issue. cmd /c "git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 2>&1" - task: PowerShell@2 From 883c1e3253b5582f3024f02fa89580dbdb4bf9be Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Fri, 21 Feb 2020 10:30:54 +0300 Subject: [PATCH 17/22] Update comment --- images.CI/azure-pipelines/image-generation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index d4fdf0ea..51850254 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -14,8 +14,8 @@ jobs: script: | Remove-Item -path './*' -Recurse -Force Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" - # git in powershell sometimes broke the AzDO pipelines with incorrect stdout interpretation - # we redirect the git output to stdout manually to avoid this issue. + # git on self-hosted agent produces some output to stderr even during successful cloning + # use cmd output redirect to overcome it cmd /c "git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 2>&1" - task: PowerShell@2 From fb51ff3b1df62287c367ee0f1998aa2a24c52abc Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Fri, 21 Feb 2020 11:34:07 +0300 Subject: [PATCH 18/22] Change image prefix (#14) * add release step * add creation release * move file to another folder * resolve comments * minor changes * change BuildNumber to BuildId --- images.CI/azure-pipelines/image-generation.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 51850254..0e20091f 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -23,7 +23,7 @@ jobs: inputs: targetType: filePath filePath: ./images.CI/build-image.ps1 - arguments: -ResourcesNamePrefix $(Build.BuildNumber) ` + arguments: -ResourcesNamePrefix $(Build.BuildId) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` -Image ${{ parameters.image_type }} ` @@ -42,7 +42,7 @@ jobs: inputs: targetType: filePath filePath: ./images.CI/create-release.ps1 - arguments: -BuildId $(Build.BuildNumber) ` + arguments: -BuildId $(Build.BuildId) ` -Organization $(RELEASE_TARGET_ORGANIZATION) ` -DefinitionId $(RELEASE_TARGET_DEFINITION_ID) ` -Project $(RELEASE_TARGET_PROJECT) ` @@ -55,7 +55,7 @@ jobs: inputs: targetType: filePath filePath: ./images.CI/cleanup.ps1 - arguments: -ResourcesNamePrefix $(Build.BuildNumber) ` + arguments: -ResourcesNamePrefix $(Build.BuildId) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` -Image ${{ parameters.image_type }} ` From 9cc623aaa951d7d141f399d84a54f0d87b0a8ca7 Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Tue, 25 Feb 2020 19:07:45 +0300 Subject: [PATCH 19/22] Add explanation why we use Azure DevOps CI --- images.CI/azure-pipelines/image-generation.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 0e20091f..7eeb555f 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -1,3 +1,9 @@ +# Currently, we use Azure DevOps for Images.CI as a temporary solution until GitHub Actions supports our requirements +# Since we have to use self-hosted machines to run image builds, we need the following features to use GitHub Actions for Images CI: +# - https://github.community/t5/GitHub-Actions/Make-secrets-available-to-builds-of-forks/m-p/30678#M508 +# - https://github.community/t5/GitHub-Actions/GitHub-Actions-Manual-Trigger-Approvals/td-p/31504 +# - https://github.community/t5/GitHub-Actions/Protecting-github-workflows/td-p/30290 + jobs: - job: pool: ci-agent-pool @@ -8,7 +14,7 @@ jobs: steps: - task: PowerShell@2 displayName: 'Download custom repository' - condition: and(ne(variables['CUSTOM_REPOSITORY_BRANCH'], ''), ne(variables['CUSTOM_REPOSITORY_URL'], '')) + condition: and(ne(variables['CUSTOM_REPOSITORY_URL'], ''), ne(variables['CUSTOM_REPOSITORY_BRANCH'], '')) inputs: targetType: 'inline' script: | From f85d7ff2754519d0949bcc991ec47a2e0bfca2cf Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Tue, 25 Feb 2020 19:32:18 +0300 Subject: [PATCH 20/22] enable scheduling --- images.CI/azure-pipelines/ubuntu1604.yml | 14 +++++++------- images.CI/azure-pipelines/ubuntu1804.yml | 14 +++++++------- images.CI/azure-pipelines/windows2016.yml | 14 +++++++------- images.CI/azure-pipelines/windows2019.yml | 14 +++++++------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/images.CI/azure-pipelines/ubuntu1604.yml b/images.CI/azure-pipelines/ubuntu1604.yml index b19e79a1..8f6dcef2 100644 --- a/images.CI/azure-pipelines/ubuntu1604.yml +++ b/images.CI/azure-pipelines/ubuntu1604.yml @@ -1,10 +1,10 @@ -# schedules: -# - cron: "0 0 * * *" -# displayName: Daily -# branches: -# include: -# - master -# always: true +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - master + always: true trigger: none pr: diff --git a/images.CI/azure-pipelines/ubuntu1804.yml b/images.CI/azure-pipelines/ubuntu1804.yml index db8acfd9..998ba42e 100644 --- a/images.CI/azure-pipelines/ubuntu1804.yml +++ b/images.CI/azure-pipelines/ubuntu1804.yml @@ -1,10 +1,10 @@ -# schedules: -# - cron: "0 0 * * *" -# displayName: Daily -# branches: -# include: -# - master -# always: true +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - master + always: true trigger: none pr: diff --git a/images.CI/azure-pipelines/windows2016.yml b/images.CI/azure-pipelines/windows2016.yml index eaac2d1b..667b30bd 100644 --- a/images.CI/azure-pipelines/windows2016.yml +++ b/images.CI/azure-pipelines/windows2016.yml @@ -1,10 +1,10 @@ -# schedules: -# - cron: "0 0 * * *" -# displayName: Daily -# branches: -# include: -# - master -# always: true +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - master + always: true trigger: none pr: diff --git a/images.CI/azure-pipelines/windows2019.yml b/images.CI/azure-pipelines/windows2019.yml index ebbf68e7..508fa12b 100644 --- a/images.CI/azure-pipelines/windows2019.yml +++ b/images.CI/azure-pipelines/windows2019.yml @@ -1,10 +1,10 @@ -# schedules: -# - cron: "0 0 * * *" -# displayName: Daily -# branches: -# include: -# - master -# always: true +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - master + always: true trigger: none pr: From c100b2846dc285534e6827e09cd5b13b989a5ef9 Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Wed, 26 Feb 2020 11:36:04 +0300 Subject: [PATCH 21/22] resolve comments --- images.CI/azure-pipelines/image-generation.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 7eeb555f..33b29273 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -1,5 +1,5 @@ -# Currently, we use Azure DevOps for Images.CI as a temporary solution until GitHub Actions supports our requirements -# Since we have to use self-hosted machines to run image builds, we need the following features to use GitHub Actions for Images CI: +# Ideally we would use GitHub Actions for this, but since we use self-hosted machines to run image builds +# we need the following features to use GitHub Actions for Images CI: # - https://github.community/t5/GitHub-Actions/Make-secrets-available-to-builds-of-forks/m-p/30678#M508 # - https://github.community/t5/GitHub-Actions/GitHub-Actions-Manual-Trigger-Approvals/td-p/31504 # - https://github.community/t5/GitHub-Actions/Protecting-github-workflows/td-p/30290 @@ -20,9 +20,12 @@ jobs: script: | Remove-Item -path './*' -Recurse -Force Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" + $env:GIT_REDIRECT_STDERR = '2>&1' + git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 + # git # git on self-hosted agent produces some output to stderr even during successful cloning # use cmd output redirect to overcome it - cmd /c "git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 2>&1" + # cmd /c "git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 2>&1" - task: PowerShell@2 displayName: 'Build VM' From d92fbab6501ed3ef352620267725a9ac9912521c Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Wed, 26 Feb 2020 11:39:53 +0300 Subject: [PATCH 22/22] Update image-generation.yml --- images.CI/azure-pipelines/image-generation.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 33b29273..332f34ad 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -18,14 +18,11 @@ jobs: inputs: targetType: 'inline' script: | + Write-Host "Clean up default repository" Remove-Item -path './*' -Recurse -Force Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" $env:GIT_REDIRECT_STDERR = '2>&1' git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 - # git - # git on self-hosted agent produces some output to stderr even during successful cloning - # use cmd output redirect to overcome it - # cmd /c "git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 2>&1" - task: PowerShell@2 displayName: 'Build VM'