From 55d0c3bed6d1c8e1480a85a2d4965add4940b9fc Mon Sep 17 00:00:00 2001
From: Leonid Lapshin <v-llapshin@microsoft.com>
Date: Tue, 1 Dec 2020 10:26:50 +0300
Subject: [PATCH] [ubuntu] Improve dotnet SDKs installation (#2078)

* download dotnet sdk with retries

* url file doesnt exists

* changed background processes to gnu parallel

* moved scriptblock to function

* fixed path

* changed rsync to mv

* typo

* rsync with remove-source-files option

* minor function changes

Co-authored-by: Leonid Lapshin <originalnoe-nazvanie@yandex.ru>
---
 .../scripts/installers/dotnetcore-sdk.sh      | 43 ++++++++++---------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/images/linux/scripts/installers/dotnetcore-sdk.sh b/images/linux/scripts/installers/dotnetcore-sdk.sh
index 31210b6ef..b22306791 100644
--- a/images/linux/scripts/installers/dotnetcore-sdk.sh
+++ b/images/linux/scripts/installers/dotnetcore-sdk.sh
@@ -46,34 +46,35 @@ done
 sdks=()
 for version in ${versions[@]}; do
     release_url="https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/${version}/releases.json"
-    echo "${release_url}"
-    releases=$(curl "${release_url}")
+    download_with_retries "${release_url}" "." "${version}.json"
+    releases=$(cat "./${version}.json")
     sdks=("${sdks[@]}" $(echo "${releases}" | jq '.releases[]' | jq '.sdk.version'))
     sdks=("${sdks[@]}" $(echo "${releases}" | jq '.releases[]' | jq '.sdks[]?' | jq '.version'))
+    rm ./${version}.json
 done
 
 sortedSdks=$(echo ${sdks[@]} | tr ' ' '\n' | grep -v preview | grep -v rc | grep -v display | cut -d\" -f2 | sort -u -r)
+extract_dotnet_sdk() {
+    local ARCHIVE_NAME="$1"
+    set -e
+    dest="./tmp-$(basename -s .tar.gz $ARCHIVE_NAME)"
+    echo "Extracting $ARCHIVE_NAME to $dest"
+    mkdir "$dest" && tar -C "$dest" -xzf "$ARCHIVE_NAME"
+    rsync -qav --remove-source-files "$dest/shared/" /usr/share/dotnet/shared/
+    rsync -qav --remove-source-files "$dest/host/" /usr/share/dotnet/host/
+    rsync -qav --remove-source-files "$dest/sdk/" /usr/share/dotnet/sdk/
+    rm -rf "$dest" "$ARCHIVE_NAME"
+}
 
-for sdk in $sortedSdks; do
-    url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$sdk/dotnet-sdk-$sdk-linux-x64.tar.gz"
-    echo "$url" >> urls
-    echo "Adding $url to list to download later"
-done
+# Download/install additional SDKs in parallel
+export -f download_with_retries
+export -f extract_dotnet_sdk
 
-# Download additional SDKs
-echo "Downloading release tarballs..."
-cat urls | xargs -n 1 -P 16 wget -q
-for tarball in *.tar.gz; do
-    dest="./tmp-$(basename -s .tar.gz $tarball)"
-    echo "Extracting $tarball to $dest"
-    mkdir "$dest" && tar -C "$dest" -xzf "$tarball"
-    rsync -qav "$dest/shared/" /usr/share/dotnet/shared/
-    rsync -qav "$dest/host/" /usr/share/dotnet/host/
-    rsync -qav "$dest/sdk/" /usr/share/dotnet/sdk/
-    rm -rf "$dest"
-    rm "$tarball"
-done
-rm urls
+parallel --jobs 0 --halt soon,fail=1 \
+    'url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/{}/dotnet-sdk-{}-linux-x64.tar.gz"; \
+    download_with_retries $url' ::: "${sortedSdks[@]}"
+
+find . -name "*.tar.gz" | parallel --halt soon,fail=1 'extract_dotnet_sdk {}'
 
 # Smoke test each SDK
 for sdk in $sortedSdks; do