From a636448540ba83c39b0f0ddb4f180d47d1c27f7b Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Sun, 20 Sep 2020 19:53:35 +0300 Subject: [PATCH 01/13] hardcode edge --- images/win/scripts/Installers/Install-Edge.ps1 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/images/win/scripts/Installers/Install-Edge.ps1 b/images/win/scripts/Installers/Install-Edge.ps1 index ebb7871b..ada5eaac 100644 --- a/images/win/scripts/Installers/Install-Edge.ps1 +++ b/images/win/scripts/Installers/Install-Edge.ps1 @@ -17,12 +17,13 @@ Write-Host "Get the Microsoft Edge WebDriver version..." $RegistryPath = "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths" $EdgePath = (Get-ItemProperty "$RegistryPath\msedge.exe").'(default)' [version]$EdgeVersion = [System.Diagnostics.FileVersionInfo]::GetVersionInfo($EdgePath).ProductVersion -$EdgeDriverVersionUrl = "https://msedgedriver.azureedge.net/LATEST_RELEASE_$($EdgeVersion.Major)" +#$EdgeDriverVersionUrl = "https://msedgedriver.azureedge.net/LATEST_RELEASE_$($EdgeVersion.Major)" -$EdgeDriverVersionFile = Start-DownloadWithRetry -Url $EdgeDriverVersionUrl -Name "versioninfo.txt" -DownloadPath $EdgeDriverPath +#$EdgeDriverVersionFile = Start-DownloadWithRetry -Url $EdgeDriverVersionUrl -Name "versioninfo.txt" -DownloadPath $EdgeDriverPath +Add-Content -Path "${EdgeDriverPath}\versioninfo.txt" -Value "85.0.564.51" Write-Host "Download Microsoft Edge WebDriver..." -$EdgeDriverLatestVersion = Get-Content -Path $EdgeDriverVersionFile +$EdgeDriverLatestVersion = Get-Content -Path "${EdgeDriverPath}\versioninfo.txt" $EdgeDriverArchName = "edgedriver_win64.zip" $EdgeDriverDownloadUrl="https://msedgedriver.azureedge.net/${EdgeDriverLatestVersion}/${EdgeDriverArchName}" From 8de9bcc26c144ed1f31e874de40e98c7007e424f Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov Date: Mon, 21 Sep 2020 10:53:56 +0300 Subject: [PATCH 02/13] add AppleWWDRCAG3.cer --- .../configuration/configure-machine.sh | 11 +++++++++- images/macos/tests/Common.Tests.ps1 | 22 +++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/images/macos/provision/configuration/configure-machine.sh b/images/macos/provision/configuration/configure-machine.sh index d34d7002..dbfae673 100644 --- a/images/macos/provision/configuration/configure-machine.sh +++ b/images/macos/provision/configuration/configure-machine.sh @@ -18,4 +18,13 @@ sudo pmset hibernatemode 0 sudo rm -f /var/vm/sleepimage # Change screen resolution to the maximum supported for 4Mb video memory -sudo "/Library/Application Support/VMware Tools/vmware-resolutionSet" 1176 885 \ No newline at end of file +sudo "/Library/Application Support/VMware Tools/vmware-resolutionSet" 1176 885 + +# https://developer.apple.com/support/expiration/ +# Enterprise iOS Distribution Certificates generated between February 7 and September 1st, 2020 will expire on February 7, 2023. +# Rotate the certificate before expiration to ensure your apps are installed and signed with an active certificate. +# Confirm that the correct intermediate certificate is installed by verifying the expiration date is set to 2030. +# sudo security delete-certificate -Z FF6797793A3CD798DC5B2ABEF56F73EDC9F83A64 /Library/Keychains/System.keychain +curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer --output $HOME/AppleWWDRCAG3.cer --silent +sudo security add-trusted-cert -d -r unspecified -k /Library/Keychains/System.keychain $HOME/AppleWWDRCAG3.cer +rm $HOME/AppleWWDRCAG3.cer \ No newline at end of file diff --git a/images/macos/tests/Common.Tests.ps1 b/images/macos/tests/Common.Tests.ps1 index ed813f58..df7d505e 100644 --- a/images/macos/tests/Common.Tests.ps1 +++ b/images/macos/tests/Common.Tests.ps1 @@ -12,6 +12,14 @@ Describe "Disk free space" { } } +Describe "Certificate" { + It "Apple Worldwide Developer Relations Certification Authority[expired: 2030-02] is installed" { + $sha1Hash = "06EC06599F4ED0027CC58956B4D3AC1255114F35" + $certs = security find-certificate -a -c Worldwide -p -Z | Out-String + $certs | Should -Match $sha1Hash + } +} + Describe "Git" { It "git is installed" { "git --version" | Should -ReturnZeroExitCode @@ -161,7 +169,7 @@ Describe "Common utilities" { It "PostgreSQL-Client" { "psql --version" | Should -ReturnZeroExitCode } - + It "PostgreSQL-Server" { "pg_config --version" | Should -ReturnZeroExitCode } @@ -178,11 +186,11 @@ Describe "Common utilities" { Get-WhichTool "php" | Should -Not -BeLike "/usr/bin/php*" "php --version" | Should -ReturnZeroExitCode } - + It "Composer" { "composer --version" | Should -ReturnZeroExitCode } - + It "R" -Skip:($os.IsBigSur) { "R --version" | Should -ReturnZeroExitCode } @@ -198,7 +206,7 @@ Describe "Common utilities" { It "bazelisk" { "bazelisk version" | Should -ReturnZeroExitCode } - + It "Julia" { "julia --version" | Should -ReturnZeroExitCode } @@ -252,7 +260,7 @@ Describe "Browsers" { It "Microsoft Edge Driver" { "msedgedriver --version" | Should -ReturnZeroExitCode } - + It "Firefox" { $firefoxLocation = "/Applications/Firefox.app/Contents/MacOS/firefox" $firefoxLocation | Should -Exist @@ -304,7 +312,7 @@ Describe "Haskell" -Skip:($os.IsHighSierra) { It "GHC" { "ghc --version" | Should -ReturnZeroExitCode } - + It "Cabal" { "cabal --version" | Should -ReturnZeroExitCode } @@ -327,7 +335,7 @@ Describe "Gcc" -Skip:($os.IsHighSierra) { param ( [string] $GccVersion ) - + "gcc-$GccVersion --version" | Should -ReturnZeroExitCode } } From d4335cc52e0569b7e5ac9720b10ddbecaede0995 Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov Date: Mon, 21 Sep 2020 11:18:28 +0300 Subject: [PATCH 03/13] set chmod -R 777 /opt --- .../scripts/installers/post-deployment.sh | 11 ++++++++ images/linux/ubuntu1604.json | 25 ++++++++++++------- images/linux/ubuntu1804.json | 25 ++++++++++++------- images/linux/ubuntu2004.json | 7 ++++++ 4 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 images/linux/scripts/installers/post-deployment.sh diff --git a/images/linux/scripts/installers/post-deployment.sh b/images/linux/scripts/installers/post-deployment.sh new file mode 100644 index 00000000..10fe34fe --- /dev/null +++ b/images/linux/scripts/installers/post-deployment.sh @@ -0,0 +1,11 @@ +#!/bin/bash +################################################################################ +## File: post-deployment.sh +## Desc: Post deployment actions +################################################################################ + +# set chmod -R 777 /opt +if [[ -d "/opt" ]]; then + echo "chmod -R 777 /opt" + chmod -R 777 /opt +fi diff --git a/images/linux/ubuntu1604.json b/images/linux/ubuntu1604.json index 33263345..111f87f0 100644 --- a/images/linux/ubuntu1604.json +++ b/images/linux/ubuntu1604.json @@ -301,15 +301,6 @@ "destination": "{{template_dir}}/Ubuntu1604-README.md", "direction": "download" }, - { - "type": "shell", - "inline": [ - "rm -rf {{user `helper_script_folder`}}", - "rm -rf {{user `installer_script_folder`}}", - "chmod 755 {{user `image_folder`}}" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, { "type": "shell", "scripts":[ @@ -319,6 +310,22 @@ "RUN_VALIDATION={{user `run_validation_diskspace`}}" ] }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/post-deployment.sh" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, + { + "type": "shell", + "inline": [ + "rm -rf {{user `helper_script_folder`}}", + "rm -rf {{user `installer_script_folder`}}", + "chmod 755 {{user `image_folder`}}" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, { "type": "file", "source": "{{template_dir}}/config/ubuntu1604.conf", diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index 1dd62bc2..d62de1e4 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -305,15 +305,6 @@ "destination": "{{template_dir}}/Ubuntu1804-README.md", "direction": "download" }, - { - "type": "shell", - "inline": [ - "rm -rf {{user `helper_script_folder`}}", - "rm -rf {{user `installer_script_folder`}}", - "chmod 755 {{user `image_folder`}}" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, { "type": "shell", "scripts":[ @@ -323,6 +314,22 @@ "RUN_VALIDATION={{user `run_validation_diskspace`}}" ] }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/post-deployment.sh" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, + { + "type": "shell", + "inline": [ + "rm -rf {{user `helper_script_folder`}}", + "rm -rf {{user `installer_script_folder`}}", + "chmod 755 {{user `image_folder`}}" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, { "type": "file", "source": "{{template_dir}}/config/ubuntu1804.conf", diff --git a/images/linux/ubuntu2004.json b/images/linux/ubuntu2004.json index 67d6e1d6..5a0f7760 100644 --- a/images/linux/ubuntu2004.json +++ b/images/linux/ubuntu2004.json @@ -316,6 +316,13 @@ "RUN_VALIDATION={{user `run_validation_diskspace`}}" ] }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/post-deployment.sh" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, { "type": "shell", "inline": [ From 6986bb8008905424d8c2b140323b841a783f97a8 Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Tue, 22 Sep 2020 10:44:51 +0300 Subject: [PATCH 04/13] set condition to install .51 instead of .60 --- images/win/scripts/Installers/Install-Edge.ps1 | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/images/win/scripts/Installers/Install-Edge.ps1 b/images/win/scripts/Installers/Install-Edge.ps1 index ada5eaac..888a9285 100644 --- a/images/win/scripts/Installers/Install-Edge.ps1 +++ b/images/win/scripts/Installers/Install-Edge.ps1 @@ -17,15 +17,21 @@ Write-Host "Get the Microsoft Edge WebDriver version..." $RegistryPath = "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths" $EdgePath = (Get-ItemProperty "$RegistryPath\msedge.exe").'(default)' [version]$EdgeVersion = [System.Diagnostics.FileVersionInfo]::GetVersionInfo($EdgePath).ProductVersion -#$EdgeDriverVersionUrl = "https://msedgedriver.azureedge.net/LATEST_RELEASE_$($EdgeVersion.Major)" +$EdgeDriverVersionUrl = "https://msedgedriver.azureedge.net/LATEST_RELEASE_$($EdgeVersion.Major)" -#$EdgeDriverVersionFile = Start-DownloadWithRetry -Url $EdgeDriverVersionUrl -Name "versioninfo.txt" -DownloadPath $EdgeDriverPath -Add-Content -Path "${EdgeDriverPath}\versioninfo.txt" -Value "85.0.564.51" +$EdgeDriverVersionFile = Start-DownloadWithRetry -Url $EdgeDriverVersionUrl -Name "versioninfo.txt" -DownloadPath $EdgeDriverPath Write-Host "Download Microsoft Edge WebDriver..." $EdgeDriverLatestVersion = Get-Content -Path "${EdgeDriverPath}\versioninfo.txt" $EdgeDriverArchName = "edgedriver_win64.zip" -$EdgeDriverDownloadUrl="https://msedgedriver.azureedge.net/${EdgeDriverLatestVersion}/${EdgeDriverArchName}" +# A temporary workaround to install the previous driver version because 85.0.564.60 for win64 doesn't exist +if ($EdgeDriverLatestVersion -eq "85.0.564.60") +{ + $EdgeDriverLatestVersion = "85.0.564.51" + Set-Content -Path "${EdgeDriverPath}\versioninfo.txt" -Value $EdgeDriverLatestVersion +} + +$EdgeDriverDownloadUrl = "https://msedgedriver.azureedge.net/${EdgeDriverLatestVersion}/${EdgeDriverArchName}" $EdgeDriverArchPath = Start-DownloadWithRetry -Url $EdgeDriverDownloadUrl -Name $EdgeDriverArchName From 3a47b604fd3117c9db6d99fcd0ffc74dd263e0c2 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Tue, 22 Sep 2020 09:46:15 +0200 Subject: [PATCH 05/13] fixes #1633 , using azure module, adjusted documentation --- help/CreateImageAndAzureResources.md | 3 ++- helpers/GenerateResourcesAndImage.ps1 | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/help/CreateImageAndAzureResources.md b/help/CreateImageAndAzureResources.md index 123d335c..3401c1fc 100644 --- a/help/CreateImageAndAzureResources.md +++ b/help/CreateImageAndAzureResources.md @@ -12,6 +12,7 @@ After successful image generation, a snapshot of the temporary VM will be conver - `packer` - Can be downloaded from https://www.packer.io/downloads - `PowerShell 5.0 or higher` or `PSCore` for linux distributes. - `Azure CLI ` - https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest +- `Azure Powershell module` - https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-4.6.1 ### Azure DevOps self-hosted pool requirements To connect to a temporary VM packer use WinRM or SSH connections on public IP interfaces. @@ -31,7 +32,7 @@ Download `packer` from https://www.packer.io/downloads, or install it via Chocol choco install packer ``` -Install Azure CLI - https://docs.microsoft.com/ru-ru/cli/azure/install-azure-cli-windows?view=azure-cli-latest&tabs=azure-cli. +Install Azure CLI - https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-windows?view=azure-cli-latest&tabs=azure-cli. ``` Invoke-WebRequest -Uri https://aka.ms/installazurecliwindows -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; rm .\AzureCLI.msi ``` diff --git a/helpers/GenerateResourcesAndImage.ps1 b/helpers/GenerateResourcesAndImage.ps1 index 32177094..31d48ee6 100644 --- a/helpers/GenerateResourcesAndImage.ps1 +++ b/helpers/GenerateResourcesAndImage.ps1 @@ -110,12 +110,12 @@ Function GenerateResourcesAndImage { $ServicePrincipalClientSecret = $env:UserName + [System.GUID]::NewGuid().ToString().ToUpper(); $InstallPassword = $env:UserName + [System.GUID]::NewGuid().ToString().ToUpper(); - Login-AzureRmAccount - Set-AzureRmContext -SubscriptionId $SubscriptionId + Connect-AzAccount + Set-AzContext -SubscriptionId $SubscriptionId $alreadyExists = $true; try { - Get-AzureRmResourceGroup -Name $ResourceGroupName + Get-AzResourceGroup -Name $ResourceGroupName Write-Verbose "Resource group was found, will delete and recreate it." } catch { @@ -126,8 +126,8 @@ Function GenerateResourcesAndImage { if ($alreadyExists) { if($Force -eq $true) { # Cleanup the resource group if it already exitsted before - Remove-AzureRmResourceGroup -Name $ResourceGroupName -Force - New-AzureRmResourceGroup -Name $ResourceGroupName -Location $AzureLocation + Remove-AzResourceGroup -Name $ResourceGroupName -Force + New-AzResourceGroup -Name $ResourceGroupName -Location $AzureLocation } else { $title = "Delete Resource Group" $message = "The resource group you specified already exists. Do you want to clean it up?" @@ -146,13 +146,13 @@ Function GenerateResourcesAndImage { switch ($result) { - 0 { Remove-AzureRmResourceGroup -Name $ResourceGroupName -Force; New-AzureRmResourceGroup -Name $ResourceGroupName -Location $AzureLocation } + 0 { Remove-AzResourceGroup -Name $ResourceGroupName -Force; New-AzResourceGroup -Name $ResourceGroupName -Location $AzureLocation } 1 { <# Do nothing #> } 2 { exit } } } } else { - New-AzureRmResourceGroup -Name $ResourceGroupName -Location $AzureLocation + New-AzResourceGroup -Name $ResourceGroupName -Location $AzureLocation } # This script should follow the recommended naming conventions for azure resources @@ -164,19 +164,19 @@ Function GenerateResourcesAndImage { $storageAccountName = $storageAccountName.Replace("-", "").Replace("_", "").Replace("(", "").Replace(")", "").ToLower() $storageAccountName += "001" - New-AzureRmStorageAccount -ResourceGroupName $ResourceGroupName -AccountName $storageAccountName -Location $AzureLocation -SkuName "Standard_LRS" + New-AzStorageAccount -ResourceGroupName $ResourceGroupName -AccountName $storageAccountName -Location $AzureLocation -SkuName "Standard_LRS" $spDisplayName = [System.GUID]::NewGuid().ToString().ToUpper() - $sp = New-AzureRmADServicePrincipal -DisplayName $spDisplayName -Password (ConvertTo-SecureString $ServicePrincipalClientSecret -AsPlainText -Force) + $sp = New-AzADServicePrincipal -DisplayName $spDisplayName -Password (ConvertTo-SecureString $ServicePrincipalClientSecret -AsPlainText -Force) $spAppId = $sp.ApplicationId $spClientId = $sp.ApplicationId $spObjectId = $sp.Id Start-Sleep -Seconds $SecondsToWaitForServicePrincipalSetup - New-AzureRmRoleAssignment -RoleDefinitionName Contributor -ServicePrincipalName $spAppId + New-AzRoleAssignment -RoleDefinitionName Contributor -ServicePrincipalName $spAppId Start-Sleep -Seconds $SecondsToWaitForServicePrincipalSetup - $sub = Get-AzureRmSubscription -SubscriptionId $SubscriptionId + $sub = Get-AzSubscription -SubscriptionId $SubscriptionId $tenantId = $sub.TenantId # "", "Note this variable-setting script for running Packer with these Azure resources in the future:", "==============================================================================================", "`$spClientId = `"$spClientId`"", "`$ServicePrincipalClientSecret = `"$ServicePrincipalClientSecret`"", "`$SubscriptionId = `"$SubscriptionId`"", "`$tenantId = `"$tenantId`"", "`$spObjectId = `"$spObjectId`"", "`$AzureLocation = `"$AzureLocation`"", "`$ResourceGroupName = `"$ResourceGroupName`"", "`$storageAccountName = `"$storageAccountName`"", "`$install_password = `"$install_password`"", "" From e5a5a6cc39933885987c09767de50d255dfba228 Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Tue, 22 Sep 2020 10:50:32 +0300 Subject: [PATCH 06/13] fix path --- images/win/scripts/Installers/Install-Edge.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/win/scripts/Installers/Install-Edge.ps1 b/images/win/scripts/Installers/Install-Edge.ps1 index 888a9285..1f4ce0ae 100644 --- a/images/win/scripts/Installers/Install-Edge.ps1 +++ b/images/win/scripts/Installers/Install-Edge.ps1 @@ -22,13 +22,13 @@ $EdgeDriverVersionUrl = "https://msedgedriver.azureedge.net/LATEST_RELEASE_$($Ed $EdgeDriverVersionFile = Start-DownloadWithRetry -Url $EdgeDriverVersionUrl -Name "versioninfo.txt" -DownloadPath $EdgeDriverPath Write-Host "Download Microsoft Edge WebDriver..." -$EdgeDriverLatestVersion = Get-Content -Path "${EdgeDriverPath}\versioninfo.txt" +$EdgeDriverLatestVersion = Get-Content -Path $EdgeDriverVersionFile $EdgeDriverArchName = "edgedriver_win64.zip" # A temporary workaround to install the previous driver version because 85.0.564.60 for win64 doesn't exist if ($EdgeDriverLatestVersion -eq "85.0.564.60") { $EdgeDriverLatestVersion = "85.0.564.51" - Set-Content -Path "${EdgeDriverPath}\versioninfo.txt" -Value $EdgeDriverLatestVersion + Set-Content -Path $EdgeDriverVersionFile -Value $EdgeDriverLatestVersion } $EdgeDriverDownloadUrl = "https://msedgedriver.azureedge.net/${EdgeDriverLatestVersion}/${EdgeDriverArchName}" From 0d663601b655b25824db345b0b20413bbd512a31 Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Tue, 22 Sep 2020 11:25:51 +0300 Subject: [PATCH 07/13] Check environment variables --- images.CI/macos/azure-pipelines/image-generation.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 490cbda6..4798c73f 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -13,6 +13,10 @@ jobs: clean: true fetchDepth: 1 + - pwsh: | + Get-ChildItem Env: + exit 1 + - task: PowerShell@2 displayName: 'Validate contributor permissions' condition: startsWith(variables['Build.SourceBranch'], 'refs/pull/') From d0524e9381ba12bb4a7c8ee8173bd7e9e8c7ea43 Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov Date: Tue, 22 Sep 2020 12:12:44 +0300 Subject: [PATCH 08/13] exclude R and virtualbox --- images/macos/provision/core/commonutils.sh | 19 +++++++++++++------ .../SoftwareReport.Generator.ps1 | 16 +++++++++++----- images/macos/tests/Common.Tests.ps1 | 18 +++++++++--------- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/images/macos/provision/core/commonutils.sh b/images/macos/provision/core/commonutils.sh index 2f2a18ce..7bd55dfd 100644 --- a/images/macos/provision/core/commonutils.sh +++ b/images/macos/provision/core/commonutils.sh @@ -40,12 +40,19 @@ for package in ${binst_common_utils[@]}; do done # brew cask install -bcask_common_utils=( - julia - virtualbox - vagrant - r -) +if is_BigSur; then + bcask_common_utils=( + julia + vagrant + ) +else + bcask_common_utils=( + julia + virtualbox + vagrant + r + ) +fi for package in ${bcask_common_utils[@]}; do echo "Install $package" diff --git a/images/macos/software-report/SoftwareReport.Generator.ps1 b/images/macos/software-report/SoftwareReport.Generator.ps1 index b8a5283c..eed6c06e 100644 --- a/images/macos/software-report/SoftwareReport.Generator.ps1 +++ b/images/macos/software-report/SoftwareReport.Generator.ps1 @@ -55,6 +55,10 @@ if ( -not $os.IsHighSierra) { $markdown += New-MDList -Style Unordered -NoNewLine -Lines $lines } +if ($os.IsLessThanBigSur) { + $markdown += New-MDList -Style Unordered -Lines @(Get-RVersion) -NoNewLine +} + $markdown += New-MDList -Style Unordered -Lines @( "Node.js ${nodejsVersion}" "NVM ${nvmVersion}" @@ -63,7 +67,6 @@ $markdown += New-MDList -Style Unordered -Lines @( $python3Version, "Ruby ${rubyVersion}", (Get-DotnetVersionList), - (Get-RVersion), "Go ${goVersion}", "$phpVersion", "$juliaVersion" @@ -139,7 +142,6 @@ $bazelVersion = Run-Command "bazel --version" | Take-Part -Part 0 -Delimiter "-" $bazeliskVersion = Run-Command "bazelisk version" | Select-String "Bazelisk version:" | Take-Part -Part 1 -Delimiter ":" $packerVersion = Run-Command "packer --version" $helmVersion = Run-Command "helm version --short" -$vbox = Run-Command "vboxmanage -v" $vagrant = Run-Command "vagrant -v" $mongo = Run-Command "mongo --version" | Select-String "MongoDB shell version" | Take-Part -Part 3 $mongod = Run-Command "mongod --version" | Select-String "db version " | Take-Part -Part 2 @@ -177,8 +179,12 @@ if ($os.IsHigherThanMojave) { $markdown += New-MDList -Lines "Newman $newmanVersion" -Style Unordered -NoNewLine } if ($os.IsLessThanBigSur) { + $vbox = Run-Command "vboxmanage -v" $parallelVersion = Run-Command "parallel --version" | Select-String "GNU parallel" | Select-Object -First 1 - $markdown += New-MDList -Lines $parallelVersion -Style Unordered + $markdown += New-MDList -Style Unordered -Lines @( + $vbox + $parallelVersion + ) } $markdown += New-MDNewLine @@ -190,9 +196,9 @@ $azureCLIVersion = Run-Command "az -v" | Select-String "^azure-cli" | Take-Part $awsVersion = Run-Command "aws --version" | Take-Part -Part 0 | Take-Part -Delimiter "/" -Part 1 $aliyunVersion = Run-Command "aliyun --version" | Select-String "Alibaba Cloud Command Line Interface Version " | Take-Part -Part 6 $awsSamVersion = Run-Command "sam --version" | Take-Part -Part 3 -$awsSessionManagerVersion = Run-Command "session-manager-plugin --version" +$awsSessionManagerVersion = Run-Command "session-manager-plugin --version" $ghcUpVersion = Run-Command "ghcup --version" | Take-Part -Part 5 -$ghcVersion = Run-Command "ghc --version" | Take-Part -Part 7 +$ghcVersion = Run-Command "ghc --version" | Take-Part -Part 7 $cabalVersion = Run-Command "cabal --version" | Take-Part -Part 3 $stackVersion = Run-Command "stack --version" | Take-Part -Part 1 | ForEach-Object {$_.replace(",","")} diff --git a/images/macos/tests/Common.Tests.ps1 b/images/macos/tests/Common.Tests.ps1 index 3e61a9a8..505c5736 100644 --- a/images/macos/tests/Common.Tests.ps1 +++ b/images/macos/tests/Common.Tests.ps1 @@ -103,7 +103,7 @@ Describe "Common utilities" { $result = Get-CommandResult "gem list" $result.Output | Should -BeLike "*nomad-cli*" } - + It "Nomad CLI IPA" { "ipa --version" | Should -ReturnZeroExitCode } @@ -163,7 +163,7 @@ Describe "Common utilities" { It "PostgreSQL-Client" { "psql --version" | Should -ReturnZeroExitCode } - + It "PostgreSQL-Server" { "pg_config --version" | Should -ReturnZeroExitCode } @@ -180,11 +180,11 @@ Describe "Common utilities" { Get-WhichTool "php" | Should -Not -BeLike "/usr/bin/php*" "php --version" | Should -ReturnZeroExitCode } - + It "Composer" { "composer --version" | Should -ReturnZeroExitCode } - + It "R" -Skip:($os.IsBigSur) { "R --version" | Should -ReturnZeroExitCode } @@ -200,7 +200,7 @@ Describe "Common utilities" { It "bazelisk" { "bazelisk version" | Should -ReturnZeroExitCode } - + It "Julia" { "julia --version" | Should -ReturnZeroExitCode } @@ -213,7 +213,7 @@ Describe "Common utilities" { "helm version --short" | Should -ReturnZeroExitCode } - It "virtualbox" { + It "virtualbox" -Skip:($os.IsBigSur) { "vboxmanage -v" | Should -ReturnZeroExitCode } @@ -254,7 +254,7 @@ Describe "Browsers" { It "Microsoft Edge Driver" { "msedgedriver --version" | Should -ReturnZeroExitCode } - + It "Firefox" { $firefoxLocation = "/Applications/Firefox.app/Contents/MacOS/firefox" $firefoxLocation | Should -Exist @@ -306,7 +306,7 @@ Describe "Haskell" -Skip:($os.IsHighSierra) { It "GHC" { "ghc --version" | Should -ReturnZeroExitCode } - + It "Cabal" { "cabal --version" | Should -ReturnZeroExitCode } @@ -329,7 +329,7 @@ Describe "Gcc" -Skip:($os.IsHighSierra) { param ( [string] $GccVersion ) - + "gcc-$GccVersion --version" | Should -ReturnZeroExitCode } } From 907a61cd2fb01af3aed01d1f66148582163c11d2 Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov Date: Tue, 22 Sep 2020 12:30:15 +0300 Subject: [PATCH 09/13] exclude vagrant --- images/macos/provision/core/commonutils.sh | 1 - images/macos/software-report/SoftwareReport.Generator.ps1 | 7 +++---- images/macos/tests/Common.Tests.ps1 | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/images/macos/provision/core/commonutils.sh b/images/macos/provision/core/commonutils.sh index 7bd55dfd..439a0106 100644 --- a/images/macos/provision/core/commonutils.sh +++ b/images/macos/provision/core/commonutils.sh @@ -43,7 +43,6 @@ done if is_BigSur; then bcask_common_utils=( julia - vagrant ) else bcask_common_utils=( diff --git a/images/macos/software-report/SoftwareReport.Generator.ps1 b/images/macos/software-report/SoftwareReport.Generator.ps1 index eed6c06e..9a8ada4a 100644 --- a/images/macos/software-report/SoftwareReport.Generator.ps1 +++ b/images/macos/software-report/SoftwareReport.Generator.ps1 @@ -142,7 +142,6 @@ $bazelVersion = Run-Command "bazel --version" | Take-Part -Part 0 -Delimiter "-" $bazeliskVersion = Run-Command "bazelisk version" | Select-String "Bazelisk version:" | Take-Part -Part 1 -Delimiter ":" $packerVersion = Run-Command "packer --version" $helmVersion = Run-Command "helm version --short" -$vagrant = Run-Command "vagrant -v" $mongo = Run-Command "mongo --version" | Select-String "MongoDB shell version" | Take-Part -Part 3 $mongod = Run-Command "mongod --version" | Select-String "db version " | Take-Part -Part 2 $p7zip = Run-Command "7z i" | Select-String "7-Zip" | Take-Part -Part 0,2 @@ -168,10 +167,8 @@ $markdown += New-MDList -Style Unordered -NoNewLine -Lines @( $bazelVersion, "bazelisk $($bazeliskVersion.Trim())", "helm $helmVersion", - "virtualbox $vbox", "mongo $mongo", "mongod $mongod", - "$vagrant", $p7zip ) if ($os.IsHigherThanMojave) { @@ -179,10 +176,12 @@ if ($os.IsHigherThanMojave) { $markdown += New-MDList -Lines "Newman $newmanVersion" -Style Unordered -NoNewLine } if ($os.IsLessThanBigSur) { + $vagrant = Run-Command "vagrant -v" $vbox = Run-Command "vboxmanage -v" $parallelVersion = Run-Command "parallel --version" | Select-String "GNU parallel" | Select-Object -First 1 $markdown += New-MDList -Style Unordered -Lines @( - $vbox + "virtualbox $vbox", + $vagrant, $parallelVersion ) } diff --git a/images/macos/tests/Common.Tests.ps1 b/images/macos/tests/Common.Tests.ps1 index 505c5736..b6c5b099 100644 --- a/images/macos/tests/Common.Tests.ps1 +++ b/images/macos/tests/Common.Tests.ps1 @@ -217,7 +217,7 @@ Describe "Common utilities" { "vboxmanage -v" | Should -ReturnZeroExitCode } - It "vagrant" { + It "vagrant" -Skip:($os.IsBigSur) { "vagrant --version" | Should -ReturnZeroExitCode } From adf9c70bd92b5004b4f10ca9a3e93b35366655d0 Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov Date: Tue, 22 Sep 2020 12:32:31 +0300 Subject: [PATCH 10/13] package condition --- images/macos/provision/core/commonutils.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/images/macos/provision/core/commonutils.sh b/images/macos/provision/core/commonutils.sh index 439a0106..9d05bcfb 100644 --- a/images/macos/provision/core/commonutils.sh +++ b/images/macos/provision/core/commonutils.sh @@ -40,13 +40,11 @@ for package in ${binst_common_utils[@]}; do done # brew cask install -if is_BigSur; then - bcask_common_utils=( - julia - ) -else - bcask_common_utils=( - julia +bcask_common_utils=( + julia +) +if is_Less_BigSur; then + bcask_common_utils+=( virtualbox vagrant r From 49763bcc9da89d62b4ebf4fdfee4abdd5c870c43 Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov Date: Tue, 22 Sep 2020 12:33:10 +0300 Subject: [PATCH 11/13] new line --- images/macos/provision/core/commonutils.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/images/macos/provision/core/commonutils.sh b/images/macos/provision/core/commonutils.sh index 9d05bcfb..9b32d5c2 100644 --- a/images/macos/provision/core/commonutils.sh +++ b/images/macos/provision/core/commonutils.sh @@ -43,6 +43,7 @@ done bcask_common_utils=( julia ) + if is_Less_BigSur; then bcask_common_utils+=( virtualbox From 9a48139d0e372c299f06972b43d035f91a9550c3 Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Tue, 22 Sep 2020 12:54:47 +0300 Subject: [PATCH 12/13] add attempts --- images.CI/macos/azure-pipelines/image-generation.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/images.CI/macos/azure-pipelines/image-generation.yml b/images.CI/macos/azure-pipelines/image-generation.yml index 4798c73f..7f02d90f 100644 --- a/images.CI/macos/azure-pipelines/image-generation.yml +++ b/images.CI/macos/azure-pipelines/image-generation.yml @@ -7,16 +7,14 @@ jobs: variables: - group: Mac-Cloud Image Generation - group: Mac-Cloud Image Generation Key Vault + - name: VirtualMachineName + value: $(Build.BuildNumber).$(System.JobAttempt) steps: - checkout: self clean: true fetchDepth: 1 - - pwsh: | - Get-ChildItem Env: - exit 1 - - task: PowerShell@2 displayName: 'Validate contributor permissions' condition: startsWith(variables['Build.SourceBranch'], 'refs/pull/') @@ -60,7 +58,7 @@ jobs: -var="output_folder=$(output-folder)" ` -var="vm_username=$(vm-username)" ` -var="vm_password=$(vm-password)" ` - -var="build_id=$(Build.BuildNumber)" ` + -var="build_id=${{ variables.VirtualMachineName }}" ` -var="baseimage_name=${{ parameters.base_image_name }}" ` -var="github_feed_token=$(github-feed-token)" ` -var="xcode_install_user=$(xcode-installation-user)" ` @@ -87,7 +85,7 @@ jobs: ls $(Common.TestResultsDirectory) echo "Put VM name to 'VM_Done_Name' file" - echo "$(Build.BuildNumber)" > "$(Build.ArtifactStagingDirectory)/VM_Done_Name" + echo "${{ variables.VirtualMachineName }}" > "$(Build.ArtifactStagingDirectory)/VM_Done_Name" displayName: Prepare artifact - bash: | From fe9f74e51ed37a716f61a7e9de6286bf57ef55c8 Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov Date: Tue, 22 Sep 2020 13:55:21 +0300 Subject: [PATCH 13/13] rm helper and installer folder in post-deployment --- .../scripts/installers/post-deployment.sh | 5 ++++ images/linux/ubuntu1604.json | 28 ++++++++----------- images/linux/ubuntu1804.json | 28 ++++++++----------- images/linux/ubuntu2004.json | 28 ++++++++----------- 4 files changed, 41 insertions(+), 48 deletions(-) diff --git a/images/linux/scripts/installers/post-deployment.sh b/images/linux/scripts/installers/post-deployment.sh index 10fe34fe..2864dae4 100644 --- a/images/linux/scripts/installers/post-deployment.sh +++ b/images/linux/scripts/installers/post-deployment.sh @@ -9,3 +9,8 @@ if [[ -d "/opt" ]]; then echo "chmod -R 777 /opt" chmod -R 777 /opt fi + +# remove installer and helper folders +rm -rf $HELPER_SCRIPT_FOLDER +rm -rf $INSTALLER_SCRIPT_FOLDER +chmod 755 $IMAGE_FOLDER diff --git a/images/linux/ubuntu1604.json b/images/linux/ubuntu1604.json index 111f87f0..3aa477cb 100644 --- a/images/linux/ubuntu1604.json +++ b/images/linux/ubuntu1604.json @@ -301,6 +301,18 @@ "destination": "{{template_dir}}/Ubuntu1604-README.md", "direction": "download" }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/post-deployment.sh" + ], + "environment_vars":[ + "HELPER_SCRIPT_FOLDER={{user `helper_script_folder`}}", + "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}", + "IMAGE_FOLDER={{user `image_folder`}}" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, { "type": "shell", "scripts":[ @@ -310,22 +322,6 @@ "RUN_VALIDATION={{user `run_validation_diskspace`}}" ] }, - { - "type": "shell", - "scripts":[ - "{{template_dir}}/scripts/installers/post-deployment.sh" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, - { - "type": "shell", - "inline": [ - "rm -rf {{user `helper_script_folder`}}", - "rm -rf {{user `installer_script_folder`}}", - "chmod 755 {{user `image_folder`}}" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, { "type": "file", "source": "{{template_dir}}/config/ubuntu1604.conf", diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index d62de1e4..a5b71c07 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -305,6 +305,18 @@ "destination": "{{template_dir}}/Ubuntu1804-README.md", "direction": "download" }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/post-deployment.sh" + ], + "environment_vars":[ + "HELPER_SCRIPT_FOLDER={{user `helper_script_folder`}}", + "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}", + "IMAGE_FOLDER={{user `image_folder`}}" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, { "type": "shell", "scripts":[ @@ -314,22 +326,6 @@ "RUN_VALIDATION={{user `run_validation_diskspace`}}" ] }, - { - "type": "shell", - "scripts":[ - "{{template_dir}}/scripts/installers/post-deployment.sh" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, - { - "type": "shell", - "inline": [ - "rm -rf {{user `helper_script_folder`}}", - "rm -rf {{user `installer_script_folder`}}", - "chmod 755 {{user `image_folder`}}" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, { "type": "file", "source": "{{template_dir}}/config/ubuntu1804.conf", diff --git a/images/linux/ubuntu2004.json b/images/linux/ubuntu2004.json index 5a0f7760..c2a1bd56 100644 --- a/images/linux/ubuntu2004.json +++ b/images/linux/ubuntu2004.json @@ -307,6 +307,18 @@ "destination": "{{template_dir}}/Ubuntu2004-README.md", "direction": "download" }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/post-deployment.sh" + ], + "environment_vars":[ + "HELPER_SCRIPT_FOLDER={{user `helper_script_folder`}}", + "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}", + "IMAGE_FOLDER={{user `image_folder`}}" + ], + "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" + }, { "type": "shell", "scripts":[ @@ -316,22 +328,6 @@ "RUN_VALIDATION={{user `run_validation_diskspace`}}" ] }, - { - "type": "shell", - "scripts":[ - "{{template_dir}}/scripts/installers/post-deployment.sh" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, - { - "type": "shell", - "inline": [ - "rm -rf {{user `helper_script_folder`}}", - "rm -rf {{user `installer_script_folder`}}", - "chmod 755 {{user `image_folder`}}" - ], - "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" - }, { "type": "file", "source": "{{template_dir}}/config/ubuntu2004.conf",