[ubuntu] Refactor PowerShell build scripts (#9064)

* [ubuntu] Refactor PowerShell build scripts

* Add Module import

* Add Invoke-DownloadWithRetry function

* Fix temp download dir

* Update function to Add-GlobalEnvironmentVariable

images/ubuntu/scripts/build/Configure-Toolset.ps1

@@ -4,91 +4,79 @@
 ##  Desc:  Configure toolset
 ################################################################################
 
-Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1" -DisableNameChecking
+Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1"
 
-function Get-ToolsetToolFullPath
-{
-    param
-    (
-        [Parameter(Mandatory)] [string] $ToolName,
-        [Parameter(Mandatory)] [string] $ToolVersion,
-        [Parameter(Mandatory)] [string] $ToolArchitecture
+function Get-TCToolVersionPath {
+    param(
+        [Parameter(Mandatory)]
+        [string] $ToolName,
+        [Parameter(Mandatory)]
+        [string] $ToolVersion,
+        [Parameter(Mandatory)]
+        [string] $ToolArchitecture
     )
 
-    $toolPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath $toolName
-    $toolPathVersion = Join-Path -Path $toolPath -ChildPath $toolVersion
-    $foundVersion = Get-Item $toolPathVersion | Sort-Object -Property {[version]$_.name} -Descending | Select-Object -First 1
-    $installationDir = Join-Path -Path $foundVersion -ChildPath $toolArchitecture
+    $toolPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath $ToolName
+    $toolPathVersion = Join-Path -Path $toolPath -ChildPath $ToolVersion
+    $foundVersion = Get-Item $toolPathVersion | Sort-Object -Property { [version] $_.name } -Descending | Select-Object -First 1
+    $installationDir = Join-Path -Path $foundVersion -ChildPath $ToolArchitecture
+
     return $installationDir
 }
 
-function Add-EnvironmentVariable
-{
-    param
-    (
-        [Parameter(Mandatory)] [string] $Name,
-        [Parameter(Mandatory)] [string] $Value,
+function Add-GlobalEnvironmentVariable {
+    param(
+        [Parameter(Mandatory)]
+        [string] $Name,
+        [Parameter(Mandatory)]
+        [string] $Value,
         [string] $FilePath = "/etc/environment"
     )
 
-    $envVar = "{0}={1}" -f $name, $value
-    Tee-Object -InputObject $envVar -FilePath $filePath -Append
+    $envVar = "{0}={1}" -f $Name, $Value
+    Tee-Object -InputObject $envVar -FilePath $FilePath -Append
 }
 
 $ErrorActionPreference = "Stop"
 
-Write-Host "Configure toolset tools environment..."
-$toolsEnvironment = @{
+Write-Host "Configure toolcache tools environment..."
+$toolEnvConfigs = @{
     go = @{
-        command = "ln -s {0}/bin/* /usr/bin/"
+        command          = "ln -s {0}/bin/* /usr/bin/"
         variableTemplate = "GOROOT_{0}_{1}_X64"
     }
 }
 
-$toolset = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw | ConvertFrom-Json
+# Get toolcache content from toolset
+$tools = (Get-ToolsetContent).toolcache | Where-Object { $toolEnvConfigs.Keys -contains $_.name }
 
-foreach ($tool in $toolset.toolcache)
-{
-    $toolName = $tool.name
-    $toolArch = $tool.arch
-    $toolEnvironment = $toolsEnvironment[$toolName]
+foreach ($tool in $tools) {
+    $toolEnvConfig = $toolEnvConfigs[$tool.name]
 
-    if (-not $toolEnvironment)
-    {
-        continue
-    }
+    if (-not ([string]::IsNullOrEmpty($toolEnvConfig.variableTemplate))) {
+        foreach ($toolVersion in $tool.versions) {
+            Write-Host "Set $($tool.name) $toolVersion environment variable..."
+            $toolPath = Get-TCToolVersionPath -ToolName $tool.name -ToolVersion $toolVersion -ToolArchitecture $tool.arch
+            $envVariableName = $toolEnvConfig.variableTemplate -f $toolVersion.split(".")
 
-    foreach ($toolVersion in $tool.versions)
-    {
-        Write-Host "Set $toolName $toolVersion environment variable..."
-        $toolPath = Get-ToolsetToolFullPath -ToolName $toolName -ToolVersion $toolVersion -ToolArchitecture $toolArch
-        $envName = $toolEnvironment.variableTemplate -f $toolVersion.split(".")
-
-        # Add environment variable name=value
-        Add-EnvironmentVariable -Name $envName -Value $toolPath
+            Add-GlobalEnvironmentVariable -Name $envVariableName -Value $toolPath
+        }
     }
 
     # Invoke command and add env variable for the default tool version
-    $toolDefVersion = $tool.default
-    if (-not $toolDefVersion)
-    {
-        continue
-    }
+    if (-not ([string]::IsNullOrEmpty($tool.default))) {
+        $toolDefaultPath = Get-TCToolVersionPath -ToolName $tool.name -ToolVersion $tool.default -ToolArchitecture $tool.arch
 
-    $envDefName = $toolEnvironment.defaultVariable
-    $toolPath = Get-ToolsetToolFullPath -ToolName $toolName -ToolVersion $toolDefVersion -ToolArchitecture $toolArch
+        if (-not ([string]::IsNullOrEmpty($toolEnvConfig.defaultVariable))) {
+            Write-Host "Set default $($toolEnvConfig.defaultVariable) for $($tool.name) $($tool.default) environment variable..."
+            Add-GlobalEnvironmentVariable -Name $toolEnvConfig.defaultVariable -Value $toolDefaultPath
+        }
 
-    if ($envDefName)
-    {
-        Write-Host "Set default $envDefName for $toolName $toolDefVersion environment variable..."
-        Add-EnvironmentVariable -Name $envDefName -Value $toolPath
-    }
-
-    if ($toolEnvironment.command)
-    {
-        $command = $toolEnvironment.command -f $toolPath
-        Write-Host "Invoke $command command for default $toolName $toolDefVersion..."
-        Invoke-Expression -Command $command
+        if (-not ([string]::IsNullOrEmpty($toolEnvConfig.command))) {
+            $command = $toolEnvConfig.command -f $toolDefaultPath
+            Write-Host "Invoke $command command for default $($tool.name) $($tool.default) ..."
+            Invoke-Expression -Command $command
+        }
     }
 }
 

images/ubuntu/scripts/build/Install-PowerShellAzModules.ps1

@@ -6,18 +6,17 @@
 $ErrorActionPreference = "Stop"
 $ProgressPreference = "SilentlyContinue"
 
-Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1" -DisableNameChecking
+Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1"
 
 # Get modules content from toolset
 $modules = (Get-ToolsetContent).azureModules
 $installPSModulePath = "/usr/share"
 
-foreach ($module in $modules)
-{
+foreach ($module in $modules) {
     $moduleName = $module.name
+
     Write-Host "Installing ${moduleName} to the ${installPSModulePath} path..."
-    foreach ($version in $module.versions)
-    {
+    foreach ($version in $module.versions) {
         $modulePath = Join-Path -Path $installPSModulePath -ChildPath "${moduleName}_${version}"
         Write-Host " - $version [$modulePath]"
         Save-Module -Path $modulePath -Name $moduleName -RequiredVersion $version -Force
@@ -28,17 +27,16 @@ foreach ($module in $modules)
     # Get github release asset for each version
     foreach ($toolVersion in $module.zip_versions) {
         $asset = $assets | Where-Object version -eq $toolVersion `
-        | Select-Object -ExpandProperty files `
-        | Select-Object -First 1
+            | Select-Object -ExpandProperty files `
+            | Select-Object -First 1
 
-        Write-Host "Installing $($module.name) $toolVersion ..."
-        if ($null -ne $asset) {
-            Write-Host "Download $($asset.filename)"
-            wget $asset.download_url -nv --retry-connrefused --tries=10 -P $installPSModulePath
-        } else {
-            Write-Host "Asset was not found in versions manifest"
+        if (-not $asset) {
+            Write-Host "Asset for ${moduleName} ${toolVersion} was not found in versions manifest"
             exit 1
         }
+
+        Write-Host "Downloading asset for ${moduleName} ${toolVersion}: $($asset.filename)"
+        Invoke-DownloadWithRetry $asset.download_url -Destination "$installPSModulePath/$($asset.filename)"
     }
 }
 

images/ubuntu/scripts/build/Install-PowerShellModules.ps1

@@ -6,7 +6,7 @@
 $ErrorActionPreference = "Stop"
 $ProgressPreference = "SilentlyContinue"
 
-Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1" -DisableNameChecking
+Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1"
 
 # Specifies the installation policy
 Set-PSRepository -InstallationPolicy Trusted -Name PSGallery
@@ -18,22 +18,18 @@ Update-Module -Name PowerShellGet -Force
 # Install PowerShell modules
 $modules = (Get-ToolsetContent).powershellModules
 
-foreach($module in $modules)
-{
+foreach($module in $modules) {
     $moduleName = $module.name
-    Write-Host "Installing ${moduleName} module"
 
-    if ($module.versions)
-    {
-        foreach ($version in $module.versions)
-        {
+    Write-Host "Installing ${moduleName} module"
+    if ($module.versions) {
+        foreach ($version in $module.versions) {
             Write-Host " - $version"
             Install-Module -Name $moduleName -RequiredVersion $version -Scope AllUsers -SkipPublisherCheck -Force
         }
-        continue
+    } else {
+        Install-Module -Name $moduleName -Scope AllUsers -SkipPublisherCheck -Force
     }
-
-    Install-Module -Name $moduleName -Scope AllUsers -SkipPublisherCheck -Force
 }
 
 Invoke-PesterTests -TestFile "PowerShellModules" -TestName "PowerShellModules"

images/ubuntu/scripts/build/Install-Toolset.ps1

@@ -4,19 +4,21 @@
 ##  Desc:  Install toolset
 ################################################################################
 
-Function Install-Asset {
+Import-Module "$env:HELPER_SCRIPTS/../tests/Helpers.psm1"
+
+function Install-Asset {
     param(
         [Parameter(Mandatory = $true)]
         [object] $ReleaseAsset
     )
 
     Write-Host "Download $($ReleaseAsset.filename)"
-    wget $ReleaseAsset.download_url -nv --retry-connrefused --tries=10
+    $assetArchivePath = Invoke-DownloadWithRetry $ReleaseAsset.download_url
 
     Write-Host "Extract $($ReleaseAsset.filename) content..."
-    $assetFolderPath = Join-Path "/tmp" $($ReleaseAsset.filename)
-    New-Item -ItemType Directory -Path $assetFolderPath
-    tar -xzf $ReleaseAsset.filename -C $assetFolderPath
+    $assetFolderPath = Join-Path "/tmp" "$($ReleaseAsset.filename)-temp-dir"
+    New-Item -ItemType Directory -Path $assetFolderPath | Out-Null
+    tar -xzf $assetArchivePath -C $assetFolderPath
 
     Write-Host "Invoke installation script..."
     Push-Location -Path $assetFolderPath
@@ -26,10 +28,8 @@ Function Install-Asset {
 
 $ErrorActionPreference = "Stop"
 
-# Get toolset content
-$toolset = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw
-
-$tools = ConvertFrom-Json -InputObject $toolset | Select-Object -ExpandProperty toolcache | Where-Object {$_.url -ne $null }
+# Get toolcache content from toolset
+$tools = (Get-ToolsetContent).toolcache | Where-Object { $_.url -ne $null }
 
 foreach ($tool in $tools) {
     # Get versions manifest for current tool
@@ -38,17 +38,18 @@ foreach ($tool in $tools) {
     # Get github release asset for each version
     foreach ($toolVersion in $tool.versions) {
         $asset = $assets | Where-Object version -like $toolVersion `
-        | Select-Object -ExpandProperty files `
-        | Where-Object { ($_.platform -eq $tool.platform) -and ($_.platform_version -eq $tool.platform_version)} `
-        | Select-Object -First 1
+            | Select-Object -ExpandProperty files `
+            | Where-Object { ($_.platform -eq $tool.platform) -and ($_.platform_version -eq $tool.platform_version)} `
+            | Select-Object -First 1
 
-        Write-Host "Installing $($tool.name) $toolVersion $($tool.arch)..."
-        if ($null -ne $asset) {
-            Install-Asset -ReleaseAsset $asset
-        } else {
-            Write-Host "Asset was not found in versions manifest"
+        if (-not $asset) {
+            Write-Host "Asset for $($tool.name) $toolVersion $($tool.arch) not found in versions manifest"
             exit 1
         }
+
+        Write-Host "Installing $($tool.name) $toolVersion $($tool.arch)..."
+        Install-Asset -ReleaseAsset $asset
     }
+
     chown -R "$($env:SUDO_USER):$($env:SUDO_USER)" "/opt/hostedtoolcache/$($tool.name)"
 }

images/ubuntu/scripts/helpers/Common.Helpers.psm1

@@ -24,7 +24,7 @@ function Get-CommandResult {
         This command runs "ls -la" in bash and returns the output and exit code as hashtable.
 
     #>
-    param (
+    param(
         [Parameter(Mandatory=$true)]
         [string] $Command,
         [int[]] $ExpectedExitCode = 0,
@@ -76,3 +76,77 @@ function Get-ToolsetContent {
     $toolsetJson = Get-Content -Path $toolsetPath -Raw
     ConvertFrom-Json -InputObject $toolsetJson
 }
+
+function Invoke-DownloadWithRetry {
+    <#
+    .SYNOPSIS
+        Downloads a file from a given URL with retry functionality.
+
+    .DESCRIPTION
+        The Invoke-DownloadWithRetry function downloads a file from the specified URL
+        to the specified path. It includes retry functionality in case the download fails.
+
+    .PARAMETER Url
+        The URL of the file to download.
+
+    .PARAMETER Path
+        The path where the downloaded file will be saved. If not provided, a temporary path
+        will be used.
+
+    .EXAMPLE
+        Invoke-DownloadWithRetry -Url "https://example.com/file.zip" -Path "/usr/local/bin"
+        Downloads the file from the specified URL and saves it to the specified path.
+
+    .EXAMPLE
+        Invoke-DownloadWithRetry -Url "https://example.com/file.zip"
+        Downloads the file from the specified URL and saves it to a temporary path.
+
+    .OUTPUTS
+        The path where the downloaded file is saved.
+    #>
+    param(
+        [Parameter(Mandatory)]
+        [string] $Url,
+        [Alias("Destination")]
+        [string] $DestinationPath
+    )
+
+    if (-not $DestinationPath) {
+        $invalidChars = [IO.Path]::GetInvalidFileNameChars() -join ''
+        $re = "[{0}]" -f [RegEx]::Escape($invalidChars)
+        $fileName = [IO.Path]::GetFileName($Url) -replace $re
+
+        if ([String]::IsNullOrEmpty($fileName)) {
+            $fileName = [System.IO.Path]::GetRandomFileName()
+        }
+        $DestinationPath = Join-Path -Path "/tmp" -ChildPath $fileName
+    }
+
+    Write-Host "Downloading package from $Url to $DestinationPath..."
+
+    $interval = 30
+    $downloadStartTime = Get-Date
+    for ($retries = 20; $retries -gt 0; $retries--) {
+        try {
+            $attemptStartTime = Get-Date
+            (New-Object System.Net.WebClient).DownloadFile($Url, $DestinationPath)
+            $attemptSeconds = [math]::Round(($(Get-Date) - $attemptStartTime).TotalSeconds, 2)
+            Write-Host "Package downloaded in $attemptSeconds seconds"
+            break
+        } catch {
+            $attemptSeconds = [math]::Round(($(Get-Date) - $attemptStartTime).TotalSeconds, 2)
+            Write-Warning "Package download failed in $attemptSeconds seconds"
+            Write-Warning $_.Exception.Message
+        }
+
+        if ($retries -eq 0) {
+            $totalSeconds = [math]::Round(($(Get-Date) - $downloadStartTime).TotalSeconds, 2)
+            throw "Package download failed after $totalSeconds seconds"
+        }
+
+        Write-Warning "Waiting $interval seconds before retrying (retries left: $retries)..."
+        Start-Sleep -Seconds $interval
+    }
+
+    return $DestinationPath
+}