diff --git a/images/macos/helpers/Common.Helpers.psm1 b/images/macos/helpers/Common.Helpers.psm1 index dd8a2184b..f77a8c2b0 100644 --- a/images/macos/helpers/Common.Helpers.psm1 +++ b/images/macos/helpers/Common.Helpers.psm1 @@ -83,3 +83,48 @@ function Invoke-RestMethodWithRetry { ) Invoke-RestMethod $Url -MaximumRetryCount 10 -RetryIntervalSec 30 } + +function Start-DownloadWithRetry +{ + Param + ( + [Parameter(Mandatory)] + [string] $Url, + [string] $Name, + [string] $DownloadPath = "${env:Temp}", + [int] $Retries = 20 + ) + + if ([String]::IsNullOrEmpty($Name)) { + $Name = [IO.Path]::GetFileName($Url) + } + + $filePath = Join-Path -Path $DownloadPath -ChildPath $Name + + #Default retry logic for the package. + while ($Retries -gt 0) + { + try + { + Write-Host "Downloading package from: $Url to path $filePath ." + (New-Object System.Net.WebClient).DownloadFile($Url, $filePath) + break + } + catch + { + Write-Host "There is an error during package downloading:`n $_" + $Retries-- + + if ($Retries -eq 0) + { + Write-Host "File can't be downloaded. Please try later or check that file exists by url: $Url" + exit 1 + } + + Write-Host "Waiting 30 seconds before retrying. Retries left: $Retries" + Start-Sleep -Seconds 30 + } + } + + return $filePath +} \ No newline at end of file diff --git a/images/macos/provision/core/aws.sh b/images/macos/provision/core/aws.sh index 02009449a..f015f6fcf 100644 --- a/images/macos/provision/core/aws.sh +++ b/images/macos/provision/core/aws.sh @@ -1,7 +1,10 @@ #!/bin/bash -e -o pipefail +source ~/utils/utils.sh + echo Installing aws... -curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg" +AWSCLIURL="https://awscli.amazonaws.com/AWSCLIV2.pkg" +download_with_retries $AWSCLIURL "." sudo installer -pkg AWSCLIV2.pkg -target / rm -rf AWSCLIV2.pkg diff --git a/images/macos/provision/core/azcopy.sh b/images/macos/provision/core/azcopy.sh index 5098c8789..8325c8037 100755 --- a/images/macos/provision/core/azcopy.sh +++ b/images/macos/provision/core/azcopy.sh @@ -1,8 +1,10 @@ #!/bin/bash -e -o pipefail +source ~/utils/utils.sh + AZCOPY_DOWNLOAD_URL="https://aka.ms/downloadazcopy-v10-mac" -wget -O "$HOME/azcopy.zip" "$AZCOPY_DOWNLOAD_URL" +download_with_retries $AZCOPY_DOWNLOAD_URL "." "azcopy.zip" unzip azcopy.zip -d azcopy AZCOPY_EXTRACTED=$(echo azcopy/azcopy*) cp "$AZCOPY_EXTRACTED/azcopy" "/usr/local/bin/azcopy" diff --git a/images/macos/provision/core/node.sh b/images/macos/provision/core/node.sh index 59e727ce9..139bb8a45 100644 --- a/images/macos/provision/core/node.sh +++ b/images/macos/provision/core/node.sh @@ -11,7 +11,7 @@ if is_Less_Catalina; then echo Installing the latest Node JS 8... TMP_FILE=/tmp/node-v8.17.0.pkg NODEURL=https://nodejs.org/dist/latest-v8.x/node-v8.17.0.pkg - curl "${NODEURL}" -o "${TMP_FILE}" + download_with_retries $NODEURL "/tmp" sudo installer -pkg "${TMP_FILE}" -target / rm -rf "${TMP_FILE}" sudo chown -R $USER "/usr/local/lib/node_modules" diff --git a/images/macos/provision/core/openjdk.sh b/images/macos/provision/core/openjdk.sh index c2b22bafa..8fb36b81d 100644 --- a/images/macos/provision/core/openjdk.sh +++ b/images/macos/provision/core/openjdk.sh @@ -7,7 +7,7 @@ installAzulJDK() { local TMP_FILE=/tmp/openjdk.dmg local TMP_MOUNT=`/usr/bin/mktemp -d /tmp/zulu.XXXX` # Download dmg - curl "${URL}" -o "${TMP_FILE}" + download_with_retries $URL "/tmp" "openjdk.dmg" # Attach dmg hdiutil attach "${TMP_FILE}" -mountpoint "${TMP_MOUNT}" # Install pkg diff --git a/images/macos/provision/core/pypy.sh b/images/macos/provision/core/pypy.sh index d55d785af..8a373f4cf 100644 --- a/images/macos/provision/core/pypy.sh +++ b/images/macos/provision/core/pypy.sh @@ -3,6 +3,7 @@ ## File: pypy.sh ## Desc: Installs PyPy ################################################################################ + source ~/utils/utils.sh function InstallPyPy @@ -12,7 +13,7 @@ function InstallPyPy PACKAGE_TAR_NAME=$(echo $PACKAGE_URL | awk -F/ '{print $NF}') echo "Downloading tar archive '$PACKAGE_TAR_NAME' - '$PACKAGE_URL'" PACKAGE_TAR_TEMP_PATH="/tmp/$PACKAGE_TAR_NAME" - wget -q -O $PACKAGE_TAR_TEMP_PATH $PACKAGE_URL + download_with_retries $AZCOPY_DOWNLOAD_URL "/tmp" "PACKAGE_TAR_NAME" echo "Expand '$PACKAGE_TAR_NAME' to the /tmp folder" tar xf $PACKAGE_TAR_TEMP_PATH -C /tmp diff --git a/images/macos/provision/core/stack.sh b/images/macos/provision/core/stack.sh index 17c5ecd9a..a642d985c 100644 --- a/images/macos/provision/core/stack.sh +++ b/images/macos/provision/core/stack.sh @@ -1,5 +1,7 @@ #!/bin/bash -e -o pipefail +source ~/utils/utils.sh + echo "Get the latest Stack version..." StackRelease=$(curl -s "https://api.github.com/repos/commercialhaskell/stack/releases/latest") DownloadUrl=$(echo $StackRelease | jq -r '.assets[].browser_download_url | select(contains("osx-x86_64.tar.gz"))' | head -n 1) @@ -7,7 +9,7 @@ StackVersion=$(echo $StackRelease | jq -r '.name' | cut -c2-) StackArchive="/tmp/stack.tar.gz" echo "Download stack version $StackVersion..." -wget $DownloadUrl -O $StackArchive +download_with_retries $DownloadUrl "/tmp" "stack.tar.gz" StackToolcachePath="$AGENT_TOOLSDIRECTORY/stack/$StackVersion" DestinationPath="$StackToolcachePath/x64" diff --git a/images/macos/provision/core/toolset.ps1 b/images/macos/provision/core/toolset.ps1 index 9238b9ff4..8241f4ab5 100644 --- a/images/macos/provision/core/toolset.ps1 +++ b/images/macos/provision/core/toolset.ps1 @@ -23,7 +23,7 @@ Function Install-Asset { $assetArchivePath = Join-Path $assetFolderPath $ReleaseAsset.filename Write-Host "Download $($ReleaseAsset.filename) archive to the $assetFolderPath folder..." - wget -P $assetFolderPath $ReleaseAsset.download_url --retry-connrefused --retry-on-http-error=429,500,503 --wait=30 --no-verbose + Start-DownloadWithRetry -Url $ReleaseAsset.download_url -DownloadPath $assetFolderPath Write-Host "Extract $($ReleaseAsset.filename) content..." tar -xzf $assetArchivePath -C $assetFolderPath diff --git a/images/macos/provision/utils/xamarin-utils.sh b/images/macos/provision/utils/xamarin-utils.sh index 1017cec24..11b1f53af 100644 --- a/images/macos/provision/utils/xamarin-utils.sh +++ b/images/macos/provision/utils/xamarin-utils.sh @@ -1,5 +1,7 @@ #!/bin/bash -e -o pipefail +source ~/utils/utils.sh + # Xamarin can clean their SDKs while updating to newer versions, # so we should be able to detect it during image generation downloadAndInstallPKG() { @@ -175,7 +177,7 @@ downloadNUnitConsole() { pushd $TMPMOUNT sudo mkdir -p $NUNIT3_PATH - sudo curl -L -o nunit3.zip $NUNIT3_LOCATION + sudo download_with_retries $NUNIT3_LOCATION "." "nunit3.zip" echo "Installing NUnit 3..." sudo unzip nunit3.zip -d $NUNIT3_PATH @@ -191,7 +193,7 @@ installNuget() { echo "Installing nuget $NUGET_VERSION for Mono $MONO_VERSION" cd ${MONO_VERSIONS_PATH}/${MONO_VERSION}/lib/mono/nuget sudo mv nuget.exe nuget_old.exe - sudo curl -L -o nuget.exe $NUGET_URL + sudo download_with_retries $NUGET_URL "." "nuget.exe" sudo chmod a+x nuget.exe }