change wget and curl to retry functions

images/macos/helpers/Common.Helpers.psm1

@@ -83,3 +83,48 @@ function Invoke-RestMethodWithRetry {
     )
     Invoke-RestMethod $Url -MaximumRetryCount 10 -RetryIntervalSec 30
 }
+
+function Start-DownloadWithRetry
+{
+    Param
+    (
+        [Parameter(Mandatory)]
+        [string] $Url,
+        [string] $Name,
+        [string] $DownloadPath = "${env:Temp}",
+        [int] $Retries = 20
+    )
+
+    if ([String]::IsNullOrEmpty($Name)) {
+        $Name = [IO.Path]::GetFileName($Url)
+    }
+
+    $filePath = Join-Path -Path $DownloadPath -ChildPath $Name
+
+    #Default retry logic for the package.
+    while ($Retries -gt 0)
+    {
+        try
+        {
+            Write-Host "Downloading package from: $Url to path $filePath ."
+            (New-Object System.Net.WebClient).DownloadFile($Url, $filePath)
+            break
+        }
+        catch
+        {
+            Write-Host "There is an error during package downloading:`n $_"
+            $Retries--
+
+            if ($Retries -eq 0)
+            {
+                Write-Host "File can't be downloaded. Please try later or check that file exists by url: $Url"
+                exit 1
+            }
+
+            Write-Host "Waiting 30 seconds before retrying. Retries left: $Retries"
+            Start-Sleep -Seconds 30
+        }
+    }
+
+    return $filePath
+}

images/macos/provision/core/aws.sh

@@ -1,7 +1,10 @@
 #!/bin/bash -e -o pipefail
 
+source ~/utils/utils.sh
+
 echo Installing aws...
-curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg"
+AWSCLIURL="https://awscli.amazonaws.com/AWSCLIV2.pkg"
+download_with_retries $AWSCLIURL "."
 sudo installer -pkg AWSCLIV2.pkg -target /
 rm -rf AWSCLIV2.pkg
 

images/macos/provision/core/azcopy.sh

@@ -1,8 +1,10 @@
 #!/bin/bash -e -o pipefail
 
+source ~/utils/utils.sh
+
 AZCOPY_DOWNLOAD_URL="https://aka.ms/downloadazcopy-v10-mac"
 
-wget -O "$HOME/azcopy.zip" "$AZCOPY_DOWNLOAD_URL"
+download_with_retries $AZCOPY_DOWNLOAD_URL "." "azcopy.zip"
 unzip azcopy.zip -d azcopy
 AZCOPY_EXTRACTED=$(echo azcopy/azcopy*)
 cp "$AZCOPY_EXTRACTED/azcopy" "/usr/local/bin/azcopy"

images/macos/provision/core/node.sh

@@ -11,7 +11,7 @@ if is_Less_Catalina; then
   echo Installing the latest Node JS 8...
   TMP_FILE=/tmp/node-v8.17.0.pkg
   NODEURL=https://nodejs.org/dist/latest-v8.x/node-v8.17.0.pkg
-  curl "${NODEURL}" -o "${TMP_FILE}"
+  download_with_retries $NODEURL "/tmp"
   sudo installer -pkg "${TMP_FILE}" -target /
   rm -rf "${TMP_FILE}"
   sudo chown -R $USER "/usr/local/lib/node_modules"

images/macos/provision/core/openjdk.sh

@@ -7,7 +7,7 @@ installAzulJDK() {
     local TMP_FILE=/tmp/openjdk.dmg
     local TMP_MOUNT=`/usr/bin/mktemp -d /tmp/zulu.XXXX`
     # Download dmg
-    curl "${URL}" -o "${TMP_FILE}"
+    download_with_retries $URL "/tmp" "openjdk.dmg"
     # Attach dmg
     hdiutil attach "${TMP_FILE}" -mountpoint "${TMP_MOUNT}"
     # Install pkg

images/macos/provision/core/pypy.sh

@@ -3,6 +3,7 @@
 ##  File:  pypy.sh
 ##  Desc:  Installs PyPy
 ################################################################################
+
 source ~/utils/utils.sh
 
 function InstallPyPy
@@ -12,7 +13,7 @@ function InstallPyPy
     PACKAGE_TAR_NAME=$(echo $PACKAGE_URL | awk -F/ '{print $NF}')
     echo "Downloading tar archive '$PACKAGE_TAR_NAME' - '$PACKAGE_URL'"
     PACKAGE_TAR_TEMP_PATH="/tmp/$PACKAGE_TAR_NAME"
-    wget -q -O $PACKAGE_TAR_TEMP_PATH $PACKAGE_URL
+    download_with_retries $AZCOPY_DOWNLOAD_URL "/tmp" "PACKAGE_TAR_NAME"
 
     echo "Expand '$PACKAGE_TAR_NAME' to the /tmp folder"
     tar xf $PACKAGE_TAR_TEMP_PATH -C /tmp

images/macos/provision/core/stack.sh

@@ -1,5 +1,7 @@
 #!/bin/bash -e -o pipefail
 
+source ~/utils/utils.sh
+
 echo "Get the latest Stack version..."
 StackRelease=$(curl -s "https://api.github.com/repos/commercialhaskell/stack/releases/latest")
 DownloadUrl=$(echo $StackRelease | jq -r '.assets[].browser_download_url | select(contains("osx-x86_64.tar.gz"))' | head -n 1)
@@ -7,7 +9,7 @@ StackVersion=$(echo $StackRelease | jq -r '.name' | cut -c2-)
 StackArchive="/tmp/stack.tar.gz"
 
 echo "Download stack version $StackVersion..."
-wget $DownloadUrl -O $StackArchive
+download_with_retries $DownloadUrl "/tmp" "stack.tar.gz"
 
 StackToolcachePath="$AGENT_TOOLSDIRECTORY/stack/$StackVersion"
 DestinationPath="$StackToolcachePath/x64"

images/macos/provision/core/toolset.ps1

@@ -23,7 +23,7 @@ Function Install-Asset {
     $assetArchivePath = Join-Path $assetFolderPath $ReleaseAsset.filename
 
     Write-Host "Download $($ReleaseAsset.filename) archive to the $assetFolderPath folder..."
-    wget -P $assetFolderPath $ReleaseAsset.download_url --retry-connrefused --retry-on-http-error=429,500,503 --wait=30 --no-verbose
+    Start-DownloadWithRetry -Url $ReleaseAsset.download_url -DownloadPath $assetFolderPath
 
     Write-Host "Extract $($ReleaseAsset.filename) content..."
     tar -xzf $assetArchivePath -C $assetFolderPath

images/macos/provision/utils/xamarin-utils.sh

@@ -1,5 +1,7 @@
 #!/bin/bash -e -o pipefail
 
+source ~/utils/utils.sh
+
 # Xamarin can clean their SDKs while updating to newer versions,
 # so we should be able to detect it during image generation
 downloadAndInstallPKG() {
@@ -175,7 +177,7 @@ downloadNUnitConsole() {
     pushd $TMPMOUNT
 
     sudo mkdir -p $NUNIT3_PATH
-    sudo curl -L -o nunit3.zip $NUNIT3_LOCATION
+    sudo download_with_retries $NUNIT3_LOCATION "." "nunit3.zip"
 
     echo "Installing NUnit 3..."
     sudo unzip nunit3.zip -d $NUNIT3_PATH
@@ -191,7 +193,7 @@ installNuget() {
   echo "Installing nuget $NUGET_VERSION for Mono $MONO_VERSION"
   cd ${MONO_VERSIONS_PATH}/${MONO_VERSION}/lib/mono/nuget
   sudo mv nuget.exe nuget_old.exe
-  sudo curl -L -o nuget.exe $NUGET_URL
+  sudo download_with_retries $NUGET_URL "." "nuget.exe"
   sudo chmod a+x nuget.exe
 }