diff --git a/images/macos/scripts/build/Install-Toolset.ps1 b/images/macos/scripts/build/Install-Toolset.ps1 index e89da38e0..d5a278574 100644 --- a/images/macos/scripts/build/Install-Toolset.ps1 +++ b/images/macos/scripts/build/Install-Toolset.ps1 @@ -13,14 +13,12 @@ Function Install-Asset { [object] $ReleaseAsset ) - $assetFolderPath = Join-Path "/tmp" "$($ReleaseAsset.filename)-temp-dir" - New-Item -ItemType Directory -Path $assetFolderPath | Out-Null - $assetArchivePath = Join-Path $assetFolderPath $ReleaseAsset.filename - - Write-Host "Download $($ReleaseAsset.filename) archive to the $assetFolderPath folder..." - Start-DownloadWithRetry -Url $ReleaseAsset.download_url -DownloadPath $assetFolderPath + Write-Host "Download $($ReleaseAsset.filename) archive..." + $assetArchivePath = Invoke-DownloadWithRetry $ReleaseAsset.download_url Write-Host "Extract $($ReleaseAsset.filename) content..." + $assetFolderPath = Join-Path "/tmp" "$($ReleaseAsset.filename)-temp-dir" + New-Item -ItemType Directory -Path $assetFolderPath | Out-Null tar -xzf $assetArchivePath -C $assetFolderPath Write-Host "Invoke installation script..." diff --git a/images/macos/scripts/build/install-actions-cache.sh b/images/macos/scripts/build/install-actions-cache.sh index 4e6a73366..02a4c39dc 100644 --- a/images/macos/scripts/build/install-actions-cache.sh +++ b/images/macos/scripts/build/install-actions-cache.sh @@ -12,9 +12,9 @@ if [ ! -d $ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE ]; then mkdir -p $ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE fi -downloadUrl=$(get_github_package_download_url "actions/action-versions" "contains(\"action-versions.tar.gz\")" "latest") -echo "Downloading action-versions $downloadUrl" -download_with_retries "$downloadUrl" "/tmp" action-versions.tar.gz -tar -xzf /tmp/action-versions.tar.gz -C $ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE +download_url=$(get_github_package_download_url "actions/action-versions" "contains(\"action-versions.tar.gz\")" "latest") +echo "Downloading action-versions $download_url" +archive_path=$(download_with_retry "$download_url") +tar -xzf "$archive_path" -C "$ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE" invoke_tests "ActionArchiveCache" diff --git a/images/macos/scripts/build/install-android-sdk.sh b/images/macos/scripts/build/install-android-sdk.sh index c765ca647..179338748 100644 --- a/images/macos/scripts/build/install-android-sdk.sh +++ b/images/macos/scripts/build/install-android-sdk.sh @@ -43,19 +43,18 @@ ANDROID_NDK_MAJOR_LATEST=$(get_toolset_value '.android.ndk."versions"[-1]') # Newer version(s) require Java 11 by default # See https://github.com/actions/runner-images/issues/6960 ANDROID_HOME=$HOME/Library/Android/sdk -ANDROID_OSX_SDK_FILE=tools-macosx.zip # Download the latest command line tools so that we can accept all of the licenses. # See https://developer.android.com/studio/#command-tools cmdlineToolsVersion=$(get_toolset_value '.android."cmdline-tools"') if [[ $cmdlineToolsVersion == "latest" ]]; then - repositoryXmlUrl="https://dl.google.com/android/repository/repository2-1.xml" - download_with_retries $repositoryXmlUrl "/tmp" "repository2-1.xml" + repository_xml_url="https://dl.google.com/android/repository/repository2-1.xml" + repository_xml_path=$(download_with_retry $repository_xml_url) cmdlineToolsVersion=$( yq -p=xml \ '.sdk-repository.remotePackage[] | select(."+@path" == "cmdline-tools;latest" and .channelRef."+@ref" == "channel-0").archives.archive[].complete.url | select(contains("commandlinetools-mac"))' \ - /tmp/repository2-1.xml + "$repository_xml_path" ) if [[ -z $cmdlineToolsVersion ]]; then @@ -65,16 +64,15 @@ if [[ $cmdlineToolsVersion == "latest" ]]; then fi echo "Downloading android command line tools..." -download_with_retries "https://dl.google.com/android/repository/${cmdlineToolsVersion}" /tmp $ANDROID_OSX_SDK_FILE +archive_path=$(download_with_retry "https://dl.google.com/android/repository/${cmdlineToolsVersion}") echo "Uncompressing android command line tools..." -mkdir -p $HOME/Library/Android/sdk -unzip -q /tmp/$ANDROID_OSX_SDK_FILE -d $HOME/Library/Android/sdk/cmdline-tools -# Command line tools need to be placed in $HOME/Library/Android/sdk/cmdline-tools/latest to function properly -mv $HOME/Library/Android/sdk/cmdline-tools/cmdline-tools $HOME/Library/Android/sdk/cmdline-tools/latest -rm -f /tmp/$ANDROID_OSX_SDK_FILE +mkdir -p "$ANDROID_HOME" +unzip -q "$archive_path" -d "$ANDROID_HOME/cmdline-tools" +# Command line tools need to be placed in $ANDROID_HOME/cmdline-tools/latest to function properly +mv "$ANDROID_HOME/cmdline-tools/cmdline-tools" "$ANDROID_HOME/cmdline-tools/latest" -echo ANDROID_HOME is $ANDROID_HOME +echo ANDROID_HOME is "$ANDROID_HOME" export PATH=$PATH:$ANDROID_HOME/cmdline-tools/latest:$ANDROID_HOME/cmdline-tools/latest/bin SDKMANAGER=$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager @@ -127,12 +125,10 @@ do done # Download SDK tools to preserve backward compatibility -sdkTools="android-sdk-tools.zip" -sdkToolsVersion=$(get_toolset_value '.android."sdk-tools"') -if [ "$sdkToolsVersion" != "null" ]; then - download_with_retries "https://dl.google.com/android/repository/${sdkToolsVersion}" "." $sdkTools - unzip -o -qq $sdkTools -d ${ANDROID_SDK_ROOT} - rm -f $sdkTools +sdk_tools_version=$(get_toolset_value '.android."sdk-tools"') +if [ "$sdk_tools_version" != "null" ]; then + sdk_tools_archive_path=$(download_with_retry "https://dl.google.com/android/repository/${sdk_tools_version}") + unzip -o -qq "$sdk_tools_archive_path" -d "${ANDROID_SDK_ROOT}" fi invoke_tests "Android" diff --git a/images/macos/scripts/build/install-aws-tools.sh b/images/macos/scripts/build/install-aws-tools.sh index 98a13dcc4..113881bab 100644 --- a/images/macos/scripts/build/install-aws-tools.sh +++ b/images/macos/scripts/build/install-aws-tools.sh @@ -7,9 +7,8 @@ source ~/utils/utils.sh echo Installing aws... -AWS_CLI_URL="https://awscli.amazonaws.com/AWSCLIV2.pkg" -download_with_retries $AWS_CLI_URL "/tmp" -sudo installer -pkg /tmp/AWSCLIV2.pkg -target / +awscliv2_pkg_path=$(download_with_retry "https://awscli.amazonaws.com/AWSCLIV2.pkg") +sudo installer -pkg "$awscliv2_pkg_path" -target / echo Installing aws sam cli... brew tap aws/tap diff --git a/images/macos/scripts/build/install-azcopy.sh b/images/macos/scripts/build/install-azcopy.sh index 5cef5d415..0cd670c07 100644 --- a/images/macos/scripts/build/install-azcopy.sh +++ b/images/macos/scripts/build/install-azcopy.sh @@ -8,19 +8,14 @@ source ~/utils/utils.sh arch=$(get_arch) # Check MacOS architecture and if ARM install using brew -if [ $arch == "arm64" ]; then +if [ "$arch" == "arm64" ]; then brew_smart_install azcopy else - AZCOPY_DOWNLOAD_URL="https://aka.ms/downloadazcopy-v10-mac" - - download_with_retries $AZCOPY_DOWNLOAD_URL "/tmp" "azcopy.zip" - unzip /tmp/azcopy.zip -d azcopy - AZCOPY_EXTRACTED=$(echo azcopy/azcopy*) - cp "$AZCOPY_EXTRACTED/azcopy" "/usr/local/bin/azcopy" + archive_path=$(download_with_retry "https://aka.ms/downloadazcopy-v10-mac") + unzip -qq "$archive_path" -d /tmp/azcopy + extract_path=$(echo /tmp/azcopy/azcopy*) + cp "$extract_path/azcopy" "/usr/local/bin/azcopy" chmod +x "/usr/local/bin/azcopy" - - echo "Done, cleaning up" - rm -rf azcopy* fi invoke_tests "Common" "AzCopy" diff --git a/images/macos/scripts/build/install-chrome.sh b/images/macos/scripts/build/install-chrome.sh index c5fc50f93..22dfe4178 100644 --- a/images/macos/scripts/build/install-chrome.sh +++ b/images/macos/scripts/build/install-chrome.sh @@ -19,20 +19,18 @@ echo "Google Chrome version is $FULL_CHROME_VERSION" # Get Google Chrome versions information CHROME_PLATFORM="mac-$arch" CHROME_VERSIONS_URL="https://googlechromelabs.github.io/chrome-for-testing/latest-patch-versions-per-build-with-downloads.json" -download_with_retries "$CHROME_VERSIONS_URL" "/tmp" "latest-patch-versions-per-build-with-downloads.json" -CHROME_VERSIONS_JSON=$(cat /tmp/latest-patch-versions-per-build-with-downloads.json) +CHROME_VERSIONS_JSON="$(cat "$(download_with_retry "$CHROME_VERSIONS_URL")")" # Download and unpack the latest release of Chrome Driver CHROMEDRIVER_VERSION=$(echo "${CHROME_VERSIONS_JSON}" | jq -r '.builds["'"$CHROME_VERSION"'"].version') echo "Installing Chrome Driver version $CHROMEDRIVER_VERSION" CHROMEDRIVER_URL=$(echo "${CHROME_VERSIONS_JSON}" | jq -r '.builds["'"$CHROME_VERSION"'"].downloads.chromedriver[] | select(.platform=="'"${CHROME_PLATFORM}"'").url') -CHROMEDRIVER_ARCHIVE="chromedriver-${CHROME_PLATFORM}.zip" CHROMEDRIVER_DIR="/usr/local/share/chromedriver-${CHROME_PLATFORM}" CHROMEDRIVER_BIN="$CHROMEDRIVER_DIR/chromedriver" -download_with_retries "$CHROMEDRIVER_URL" "/tmp" "$CHROMEDRIVER_ARCHIVE" -unzip -qq /tmp/$CHROMEDRIVER_ARCHIVE -d /tmp/ +CHROMEDRIVER_ARCHIVE_PATH=$(download_with_retry "$CHROMEDRIVER_URL") +unzip -qq "$CHROMEDRIVER_ARCHIVE_PATH" -d /tmp/ sudo mv "/tmp/chromedriver-${CHROME_PLATFORM}" "$CHROMEDRIVER_DIR" ln -s "$CHROMEDRIVER_BIN" /usr/local/bin/chromedriver echo "export CHROMEWEBDRIVER=$CHROMEDRIVER_DIR" >> "${HOME}/.bashrc" @@ -42,11 +40,10 @@ CHROME_FOR_TESTING_VERSION=$(echo "${CHROME_VERSIONS_JSON}" | jq -r '.builds["'" echo "Installing Google Chrome for Testing version $CHROME_FOR_TESTING_VERSION" CHROME_FOR_TESTING_URL=$(echo "${CHROME_VERSIONS_JSON}" | jq -r '.builds["'"$CHROME_VERSION"'"].downloads.chrome[] | select(.platform=="'"${CHROME_PLATFORM}"'").url') -CHROME_FOR_TESTING_ARCHIVE="chrome-${CHROME_PLATFORM}.zip" CHROME_FOR_TESTING_APP="Google Chrome for Testing.app" -download_with_retries $CHROME_FOR_TESTING_URL "/tmp" $CHROME_FOR_TESTING_ARCHIVE -unzip -qq /tmp/$CHROME_FOR_TESTING_ARCHIVE -d /tmp/ +CHROME_FOR_TESTING_ARCHIVE_PATH=$(download_with_retry "$CHROME_FOR_TESTING_URL") +unzip -qq "$CHROME_FOR_TESTING_ARCHIVE_PATH" -d /tmp/ mv "/tmp/chrome-${CHROME_PLATFORM}/${CHROME_FOR_TESTING_APP}" "/Applications/${CHROME_FOR_TESTING_APP}" echo "Installing Selenium" diff --git a/images/macos/scripts/build/install-codeql-bundle.sh b/images/macos/scripts/build/install-codeql-bundle.sh index 8f07f72fd..68fed2634 100644 --- a/images/macos/scripts/build/install-codeql-bundle.sh +++ b/images/macos/scripts/build/install-codeql-bundle.sh @@ -7,21 +7,20 @@ source ~/utils/utils.sh # Retrieve the CLI version of the latest CodeQL bundle. -download_with_retries https://raw.githubusercontent.com/github/codeql-action/v2/src/defaults.json "/tmp" "codeql-defaults.json" -bundle_version="$(jq -r '.cliVersion' /tmp/codeql-defaults.json)" +defaults_json_path=$(download_with_retry https://raw.githubusercontent.com/github/codeql-action/v2/src/defaults.json) +bundle_version="$(jq -r '.cliVersion' "$defaults_json_path")" bundle_tag_name="codeql-bundle-v$bundle_version" echo "Downloading CodeQL bundle $bundle_version..." # Note that this is the all-platforms CodeQL bundle, to support scenarios where customers run # different operating systems within containers. -download_with_retries "https://github.com/github/codeql-action/releases/download/$bundle_tag_name/codeql-bundle.tar.gz" "/tmp" "codeql-bundle.tar.gz" -codeql_archive="/tmp/codeql-bundle.tar.gz" +archive_path=$(download_with_retry "https://github.com/github/codeql-action/releases/download/$bundle_tag_name/codeql-bundle.tar.gz") codeql_toolcache_path="$AGENT_TOOLSDIRECTORY/CodeQL/$bundle_version/x64" mkdir -p "$codeql_toolcache_path" echo "Unpacking the downloaded CodeQL bundle archive..." -tar -xzf "$codeql_archive" -C "$codeql_toolcache_path" +tar -xzf "$archive_path" -C "$codeql_toolcache_path" # Touch a file to indicate to the CodeQL Action that this bundle shipped with the toolcache. This is # to support overriding the CodeQL version specified in defaults.json on GitHub Enterprise. diff --git a/images/macos/scripts/build/install-common-utils.sh b/images/macos/scripts/build/install-common-utils.sh index 39a8acccd..253a2b60d 100644 --- a/images/macos/scripts/build/install-common-utils.sh +++ b/images/macos/scripts/build/install-common-utils.sh @@ -8,8 +8,8 @@ source ~/utils/utils.sh # Download and install YQ in cases when it is not available in the formulae as for macOS 11: https://formulae.brew.sh/formula/yq if is_BigSur; then - download_with_retries "https://github.com/mikefarah/yq/releases/latest/download/yq_darwin_amd64" "/tmp" "yq" - sudo install /tmp/yq /usr/local/bin/yq + binary_path=$(download_with_retry "https://github.com/mikefarah/yq/releases/latest/download/yq_darwin_amd64") + sudo install "$binary_path" /usr/local/bin/yq fi # Monterey needs future review: @@ -27,12 +27,10 @@ for package in $cask_packages; do if is_Monterey && [[ $package == "virtualbox" ]]; then # Do not update VirtualBox on macOS 12 due to the issue with VMs in gurumediation state which blocks Vagrant on macOS: https://github.com/actions/runner-images/issues/8730 # macOS host: Dropped all kernel extensions. VirtualBox relies fully on the hypervisor and vmnet frameworks provided by Apple now. - vbcask_url="https://raw.githubusercontent.com/Homebrew/homebrew-cask/aa3c55951fc9d687acce43e5c0338f42c1ddff7b/Casks/virtualbox.rb" - download_with_retries $vbcask_url - brew install ./virtualbox.rb - rm ./virtualbox.rb + virtualbox_cask_path=$(download_with_retry "https://raw.githubusercontent.com/Homebrew/homebrew-cask/aa3c55951fc9d687acce43e5c0338f42c1ddff7b/Casks/virtualbox.rb") + brew install "$virtualbox_cask_path" else - brew install --cask $package + brew install --cask "$package" fi done diff --git a/images/macos/scripts/build/install-dotnet.sh b/images/macos/scripts/build/install-dotnet.sh index 4ff20b764..f7fc1039d 100644 --- a/images/macos/scripts/build/install-dotnet.sh +++ b/images/macos/scripts/build/install-dotnet.sh @@ -12,8 +12,8 @@ arch=$(get_arch) # Download installer from dot.net and keep it locally DOTNET_INSTALL_SCRIPT="https://dot.net/v1/dotnet-install.sh" -download_with_retries $DOTNET_INSTALL_SCRIPT . -chmod +x ./dotnet-install.sh +install_script_path=$(download_with_retry $DOTNET_INSTALL_SCRIPT) +chmod +x "$install_script_path" ARGS_LIST=() echo "Parsing dotnet SDK (except rc and preview versions) from .json..." @@ -22,15 +22,15 @@ DOTNET_VERSIONS=($(get_toolset_value ".dotnet.arch[\"$arch\"].versions | .[]")) for DOTNET_VERSION in "${DOTNET_VERSIONS[@]}"; do RELEASE_URL="https://raw.githubusercontent.com/dotnet/core/main/release-notes/${DOTNET_VERSION}/releases.json" - download_with_retries "$RELEASE_URL" "/tmp" "dotnet_${DOTNET_VERSION}.json" + releases_json_file=$(download_with_retry "$RELEASE_URL") if [[ $DOTNET_VERSION == "6.0" ]]; then ARGS_LIST+=( - $(cat /tmp/dotnet_${DOTNET_VERSION}.json | jq -r 'first(.releases[].sdks[]?.version | select(contains("preview") or contains("rc") | not))') + $(cat "$releases_json_file" | jq -r 'first(.releases[].sdks[]?.version | select(contains("preview") or contains("rc") | not))') ) else ARGS_LIST+=( - $(cat /tmp/dotnet_${DOTNET_VERSION}.json | \ + $(cat "$releases_json_file" | \ jq -r '.releases[].sdk."version"' | grep -v -E '\-(preview|rc)\d*' | \ sort -r | rev | uniq -s 2 | rev) ) @@ -38,11 +38,9 @@ for DOTNET_VERSION in "${DOTNET_VERSIONS[@]}"; do done for ARGS in "${ARGS_LIST[@]}"; do - ./dotnet-install.sh --version $ARGS -NoPath --arch $arch + "$install_script_path" --version $ARGS -NoPath --arch $arch done -rm ./dotnet-install.sh - # dotnet installer doesn't create symlink to executable in /user/local/bin # Moreover at that moment /user/local/bin doesn't exist (though already added to $PATH) ln -s ~/.dotnet/dotnet /usr/local/bin/dotnet diff --git a/images/macos/scripts/build/install-edge.sh b/images/macos/scripts/build/install-edge.sh index d13f5b82f..94f0b7777 100644 --- a/images/macos/scripts/build/install-edge.sh +++ b/images/macos/scripts/build/install-edge.sh @@ -17,29 +17,23 @@ echo "Version of Microsoft Edge: ${EDGE_VERSION}" echo "Installing Microsoft Edge WebDriver..." -EDGE_DRIVER_VERSION_URL="https://msedgedriver.azureedge.net/LATEST_RELEASE_${EDGE_VERSION_MAJOR}_MACOS" -download_with_retries "$EDGE_DRIVER_VERSION_URL" "/tmp" "edge-version" -EDGE_DRIVER_LATEST_VERSION=$(cat /tmp/edge-version | iconv -f utf-16 -t utf-8 | tr -d '\r') +EDGE_DRIVER_VERSION_FILE_PATH=$(download_with_retry "https://msedgedriver.azureedge.net/LATEST_RELEASE_${EDGE_VERSION_MAJOR}_MACOS") +EDGE_DRIVER_LATEST_VERSION=$(iconv -f utf-16 -t utf-8 "$EDGE_DRIVER_VERSION_FILE_PATH" | tr -d '\r') EDGE_DRIVER_URL="https://msedgedriver.azureedge.net/${EDGE_DRIVER_LATEST_VERSION}/edgedriver_mac64.zip" echo "Compatible version of WebDriver: ${EDGE_DRIVER_LATEST_VERSION}" -pushd "/tmp" > /dev/null -download_with_retries $EDGE_DRIVER_URL "." "edgedriver.zip" +EDGE_DRIVER_ARCHIVE_PATH=$(download_with_retry "$EDGE_DRIVER_URL") # Move webdriver to the separate directory to be consistent with the docs # https://docs.microsoft.com/en-us/azure/devops/pipelines/test/continuous-test-selenium?view=azure-devops#decide-how-you-will-deploy-and-test-your-app -APPLICATION="/usr/local/bin/msedgedriver" -EDGEDRIVER_DIR="/usr/local/share/edge_driver" -EDGEDRIVER_BIN="$EDGEDRIVER_DIR/msedgedriver" +EDGE_DRIVER_DIR="/usr/local/share/edge_driver" +mkdir -p $EDGE_DRIVER_DIR +unzip -qq "$EDGE_DRIVER_ARCHIVE_PATH" -d "$EDGE_DRIVER_DIR" +ln -s "$EDGE_DRIVER_DIR/msedgedriver" "/usr/local/bin/msedgedriver" -mkdir -p $EDGEDRIVER_DIR - -unzip "edgedriver.zip" -d $EDGEDRIVER_DIR -ln -s "$EDGEDRIVER_BIN" $APPLICATION -echo "export EDGEWEBDRIVER=${EDGEDRIVER_DIR}" >> "${HOME}/.bashrc" -popd > /dev/null +echo "export EDGEWEBDRIVER=${EDGE_DRIVER_DIR}" >> "${HOME}/.bashrc" # Configure Edge Updater to prevent auto update # https://learn.microsoft.com/en-us/deployedge/edge-learnmore-edgeupdater-for-macos diff --git a/images/macos/scripts/build/install-homebrew.sh b/images/macos/scripts/build/install-homebrew.sh index b822c6ff1..cb32793d9 100644 --- a/images/macos/scripts/build/install-homebrew.sh +++ b/images/macos/scripts/build/install-homebrew.sh @@ -9,8 +9,8 @@ source ~/utils/utils.sh arch=$(get_arch) echo "Installing Homebrew..." -download_with_retries "https://raw.githubusercontent.com/Homebrew/install/master/install.sh" "/tmp" "homebrew-install.sh" -/bin/bash /tmp/homebrew-install.sh +homebrew_installer_path=$(download_with_retry "https://raw.githubusercontent.com/Homebrew/install/master/install.sh") +/bin/bash "$homebrew_installer_path" if [[ $arch == "arm64" ]]; then /opt/homebrew/bin/brew update diff --git a/images/macos/scripts/build/install-miniconda.sh b/images/macos/scripts/build/install-miniconda.sh index a50008f35..2ee589580 100644 --- a/images/macos/scripts/build/install-miniconda.sh +++ b/images/macos/scripts/build/install-miniconda.sh @@ -6,10 +6,9 @@ source ~/utils/utils.sh -download_with_retries "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh" "/tmp" "miniconda.sh" - -chmod +x /tmp/miniconda.sh -sudo /tmp/miniconda.sh -b -p /usr/local/miniconda +miniconda_installer_path=$(download_with_retry "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh") +chmod +x "$miniconda_installer_path" +sudo "$miniconda_installer_path" -b -p /usr/local/miniconda # Chmod with full permissions recursively to avoid permissions restrictions sudo chmod -R 777 /usr/local/miniconda diff --git a/images/macos/scripts/build/install-mono.sh b/images/macos/scripts/build/install-mono.sh index c4af70a36..cc4795144 100644 --- a/images/macos/scripts/build/install-mono.sh +++ b/images/macos/scripts/build/install-mono.sh @@ -7,43 +7,38 @@ # Source utility functions source ~/utils/utils.sh -# Create a temporary directory to store downloaded files -TMP_DIR=$(mktemp -d /tmp/visualstudio.XXXX) - # Install Mono Framework MONO_VERSION_FULL=$(get_toolset_value '.mono.framework.version') MONO_VERSION=$(echo "$MONO_VERSION_FULL" | cut -d. -f 1,2,3) MONO_VERSION_SHORT=$(echo $MONO_VERSION_FULL | cut -d. -f 1,2) MONO_PKG_URL="https://download.mono-project.com/archive/${MONO_VERSION}/macos-10-universal/MonoFramework-MDK-${MONO_VERSION_FULL}.macos10.xamarin.universal.pkg" -MONO_PKG_NAME=${MONO_PKG_URL##*/} MONO_VERSIONS_PATH='/Library/Frameworks/Mono.framework/Versions' -download_with_retries "$MONO_PKG_URL" "$TMP_DIR" -echo "Installing $MONO_PKG_NAME..." -sudo installer -pkg "$TMP_DIR/$MONO_PKG_NAME" -target / +MONO_PKG_PATH=$(download_with_retry "$MONO_PKG_URL") +echo "Installing Mono Framework ${MONO_VERSION_FULL}..." +sudo installer -pkg "$MONO_PKG_PATH" -target / # Download and install NUnit console NUNIT_VERSION=$(get_toolset_value '.mono.nunit.version') NUNIT_ARCHIVE_URL="https://github.com/nunit/nunit-console/releases/download/${NUNIT_VERSION}/NUnit.Console-${NUNIT_VERSION}.zip" -NUNIT_ARCHIVE_NAME=${NUNIT_ARCHIVE_URL##*/} NUNIT_PATH="/Library/Developer/nunit" NUNIT_VERSION_PATH="$NUNIT_PATH/$NUNIT_VERSION" -download_with_retries "$NUNIT_ARCHIVE_URL" "$TMP_DIR" -echo "Installing $NUNIT_ARCHIVE_NAME..." +NUNIT_ARCHIVE_PATH=$(download_with_retry "$NUNIT_ARCHIVE_URL") +echo "Installing NUnit ${NUNIT_VERSION}..." sudo mkdir -p "$NUNIT_VERSION_PATH" -sudo unzip -q "$TMP_DIR/$NUNIT_ARCHIVE_NAME" -d "$NUNIT_VERSION_PATH" +sudo unzip -q "$NUNIT_ARCHIVE_PATH" -d "$NUNIT_VERSION_PATH" # Create a wrapper script for nunit3-console echo "Creating nunit3-console wrapper..." -NUNIT3_CONSOLE_WRAPPER=nunit3-console -cat < "${TMP_DIR}/${NUNIT3_CONSOLE_WRAPPER}" +NUNIT3_CONSOLE_WRAPPER=$(mktemp) +cat < "$NUNIT3_CONSOLE_WRAPPER" #!/bin/bash -e -o pipefail exec ${MONO_VERSIONS_PATH}/${MONO_VERSION}/bin/mono --debug \$MONO_OPTIONS $NUNIT_VERSION_PATH/nunit3-console.exe "\$@" EOF -cat "${TMP_DIR}/${NUNIT3_CONSOLE_WRAPPER}" -sudo chmod +x "${TMP_DIR}/${NUNIT3_CONSOLE_WRAPPER}" -sudo mv "${TMP_DIR}/${NUNIT3_CONSOLE_WRAPPER}" "${MONO_VERSIONS_PATH}/${MONO_VERSION}/Commands/${NUNIT3_CONSOLE_WRAPPER}" +cat "$NUNIT3_CONSOLE_WRAPPER" +sudo chmod +x "$NUNIT3_CONSOLE_WRAPPER" +sudo mv "$NUNIT3_CONSOLE_WRAPPER" "${MONO_VERSIONS_PATH}/${MONO_VERSION}/Commands/nunit3-console" # Create a symlink for the short version of Mono (e.g., 6.12) echo "Creating short symlink '${MONO_VERSION_SHORT}'..." diff --git a/images/macos/scripts/build/install-node.sh b/images/macos/scripts/build/install-node.sh index d4ed820ef..60a25d5de 100644 --- a/images/macos/scripts/build/install-node.sh +++ b/images/macos/scripts/build/install-node.sh @@ -13,8 +13,8 @@ brew_smart_install "node@$defaultVersion" brew link node@$defaultVersion --force --overwrite echo Installing yarn... -download_with_retries "https://yarnpkg.com/install.sh" "/tmp" "yarn-install.sh" -bash /tmp/yarn-install.sh +yarn_installer_path=$(download_with_retry "https://yarnpkg.com/install.sh") +bash "$yarn_installer_path" if is_BigSur || is_Monterey; then npm_global_packages=$(get_toolset_value '.npm.global_packages[].name') diff --git a/images/macos/scripts/build/install-nvm.sh b/images/macos/scripts/build/install-nvm.sh index 59cafdaa3..4c2c121d5 100644 --- a/images/macos/scripts/build/install-nvm.sh +++ b/images/macos/scripts/build/install-nvm.sh @@ -7,25 +7,22 @@ source ~/utils/utils.sh [ -n "$API_PAT" ] && authString=(-H "Authorization: token ${API_PAT}") -VERSION=$(curl "${authString[@]}" -fsSL https://api.github.com/repos/nvm-sh/nvm/releases/latest | jq -r '.tag_name') -download_with_retries "https://raw.githubusercontent.com/nvm-sh/nvm/$VERSION/install.sh" "/tmp" "nvm-install.sh" -bash /tmp/nvm-install.sh +nvm_version=$(curl "${authString[@]}" -fsSL https://api.github.com/repos/nvm-sh/nvm/releases/latest | jq -r '.tag_name') +nvm_installer_path=$(download_with_retry "https://raw.githubusercontent.com/nvm-sh/nvm/$nvm_version/install.sh") -if [ $? -eq 0 ]; then - . ~/.bashrc - nvm --version - nodeVersions=$(get_toolset_value '.node.nvm_versions[]') - for version in ${nodeVersions[@]} - do - nvm install v${version} - done +if bash "$nvm_installer_path"; then + . ~/.bashrc + nvm --version + for version in $(get_toolset_value '.node.nvm_versions[]'); do + nvm install "v${version}" + done - # set system node as default - nvm alias default system + # set system node as default + nvm alias default system + + echo "Node version manager has been installed successfully" else - echo error + echo "Node version manager installation failed" fi -echo "Node version manager has been installed successfully" - invoke_tests "Node" "nvm" diff --git a/images/macos/scripts/build/install-openjdk.sh b/images/macos/scripts/build/install-openjdk.sh index b57877866..6000439f6 100644 --- a/images/macos/scripts/build/install-openjdk.sh +++ b/images/macos/scripts/build/install-openjdk.sh @@ -34,16 +34,16 @@ installOpenJDK() { local JAVA_VERSION=$1 # Get link for Java binaries and Java version - download_with_retries "https://api.adoptium.net/v3/assets/latest/${JAVA_VERSION}/hotspot" "/tmp" "openjdk-hotspot.json" + hotspot_json_path=$(download_with_retry "https://api.adoptium.net/v3/assets/latest/${JAVA_VERSION}/hotspot") if [[ $arch == "arm64" ]]; then - asset=$(jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="aarch64")' /tmp/openjdk-hotspot.json) + asset=$(jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="aarch64")' "$hotspot_json_path") else - asset=$(jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="x64")' /tmp/openjdk-hotspot.json) + asset=$(jq -r '.[] | select(.binary.os=="mac" and .binary.image_type=="jdk" and .binary.architecture=="x64")' "$hotspot_json_path") fi - archivePath=$(echo ${asset} | jq -r '.binary.package.link') - fullVersion=$(echo ${asset} | jq -r '.version.semver' | tr '+' '-') + archive_url=$(echo "$asset" | jq -r '.binary.package.link') + fullVersion=$(echo "$asset" | jq -r '.version.semver' | tr '+' '-') # Remove 'LTS' suffix from the version if present fullVersion="${fullVersion//.LTS/}" @@ -58,12 +58,12 @@ installOpenJDK() { fi # Download and extract Java binaries - download_with_retries ${archivePath} /tmp OpenJDK-${fullVersion}.tar.gz + archive_path=$(download_with_retry "$archive_url") echo "Creating ${javaToolcacheVersionArchPath} directory" mkdir -p ${javaToolcacheVersionArchPath} - tar -xf /tmp/OpenJDK-${fullVersion}.tar.gz -C ${javaToolcacheVersionArchPath} --strip-components=1 + tar -xf "$archive_path" -C ${javaToolcacheVersionArchPath} --strip-components=1 # Create complete file if [[ $arch == "arm64" ]]; then diff --git a/images/macos/scripts/build/install-powershell.sh b/images/macos/scripts/build/install-powershell.sh index 33adb016c..3a0a1bbd9 100644 --- a/images/macos/scripts/build/install-powershell.sh +++ b/images/macos/scripts/build/install-powershell.sh @@ -9,16 +9,15 @@ source ~/utils/utils.sh echo Installing PowerShell... arch=$(get_arch) -download_with_retries "https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/metadata.json" "/tmp" "powershell-metadata.json" -psver=$(cat /tmp/powershell-metadata.json | jq -r '.LTSReleaseTag[0]') -psDownloadUrl=$(get_github_package_download_url "PowerShell/PowerShell" "contains(\"osx-$arch.pkg\")" "$psver" "$API_PAT") -download_with_retries $psDownloadUrl "/tmp" "powershell.pkg" +metadata_json_path=$(download_with_retry "https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/metadata.json") +version=$(jq -r '.LTSReleaseTag[0]' "$metadata_json_path") +download_url=$(get_github_package_download_url "PowerShell/PowerShell" "contains(\"osx-$arch.pkg\")" "$version" "$API_PAT") +pkg_path=$(download_with_retry "$download_url") # Work around the issue on macOS Big Sur 11.5 or higher for possible error message ("can't be opened because Apple cannot check it for malicious software") when installing the package -sudo xattr -rd com.apple.quarantine /tmp/powershell.pkg +sudo xattr -rd com.apple.quarantine "$pkg_path" - -sudo installer -pkg /tmp/powershell.pkg -target / +sudo installer -pkg "$pkg_path" -target / # Install PowerShell modules psModules=$(get_toolset_value '.powershellModules[].name') diff --git a/images/macos/scripts/build/install-pypy.sh b/images/macos/scripts/build/install-pypy.sh index 39464414c..5e1900b57 100644 --- a/images/macos/scripts/build/install-pypy.sh +++ b/images/macos/scripts/build/install-pypy.sh @@ -10,13 +10,12 @@ function InstallPyPy { PACKAGE_URL=$1 - PACKAGE_TAR_NAME=$(echo $PACKAGE_URL | awk -F/ '{print $NF}') + PACKAGE_TAR_NAME=$(basename "$PACKAGE_URL") echo "Downloading tar archive '$PACKAGE_TAR_NAME'" - PACKAGE_TAR_TEMP_PATH="/tmp/$PACKAGE_TAR_NAME" - download_with_retries $PACKAGE_URL "/tmp" "$PACKAGE_TAR_NAME" + archive_path=$(download_with_retry "$PACKAGE_URL") echo "Expand '$PACKAGE_TAR_NAME' to the /tmp folder" - tar xf $PACKAGE_TAR_TEMP_PATH -C /tmp + tar xf "$archive_path" -C /tmp # Get Python version PACKAGE_NAME=${PACKAGE_TAR_NAME/.tar.bz2/} @@ -70,17 +69,14 @@ function InstallPyPy echo "Create complete file" touch $PYPY_TOOLCACHE_VERSION_PATH/x64.complete - - echo "Remove '$PACKAGE_TAR_TEMP_PATH'" - rm -f $PACKAGE_TAR_TEMP_PATH } arch=$(get_arch) -download_with_retries "https://downloads.python.org/pypy/versions.json" "/tmp" "pypy-versions.json" +versions_json_path=$(download_with_retry "https://downloads.python.org/pypy/versions.json") toolsetVersions=$(get_toolset_value '.toolcache[] | select(.name | contains("PyPy")) | .arch.'$arch'.versions[]') for toolsetVersion in $toolsetVersions; do - latestMajorPyPyVersion=$(cat /tmp/pypy-versions.json | + latestMajorPyPyVersion=$(cat "$versions_json_path" | jq -r --arg toolsetVersion $toolsetVersion '.[] | select((.python_version | startswith($toolsetVersion)) and .stable == true).files[] | select(.platform == "darwin").download_url' | head -1) diff --git a/images/macos/scripts/build/install-python.sh b/images/macos/scripts/build/install-python.sh index dd55c8204..978a19340 100644 --- a/images/macos/scripts/build/install-python.sh +++ b/images/macos/scripts/build/install-python.sh @@ -10,19 +10,19 @@ echo "Installing Python Tooling" if is_Monterey || is_BigSur; then echo "Install latest Python 2" - Python2Url="https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg" - download_with_retries $Python2Url "/tmp" "python2.pkg" + python2_pkg=$(download_with_retry "https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg") - sudo installer -showChoiceChangesXML -pkg /tmp/python2.pkg -target / > /tmp/python2_choices.xml + choice_changes_xml=$(mktemp /tmp/python2_choice_changes.xml.XXXXXX) + sudo installer -showChoiceChangesXML -pkg "$python2_pkg" -target / | tee "$choice_changes_xml" > /dev/null # To avoid symlink conflicts, remove tools installation in /usr/local/bin using installer choices - xmllint --shell /tmp/python2_choices.xml < /dev/null; - then - local ERR_EXIT_ENABLED=true - else - local ERR_EXIT_ENABLED=false - fi - set +e + echo "Downloading package from $url to $download_path..." >&2 - echo "Downloading '$URL' to '${DEST}/${NAME}'..." - retries=20 interval=30 - while [ $retries -gt 0 ]; do - ((retries--)) - test "$ERR_EXIT_ENABLED" = true && set +e - http_code=$(eval $COMMAND) - exit_code=$? - test "$ERR_EXIT_ENABLED" = true && set -e - if [ $http_code -eq 200 ] && [ $exit_code -eq 0 ]; then - echo "Download completed" - return 0 + download_start_time=$(date +%s) + + for ((retries=20; retries>0; retries--)); do + attempt_start_time=$(date +%s) + if http_code=$(curl -4sSLo "$download_path" "$url" -w '%{http_code}'); then + attempt_seconds=$(($(date +%s) - attempt_start_time)) + if [ "$http_code" -eq 200 ]; then + echo "Package downloaded in $attempt_seconds seconds" >&2 + break + else + echo "Received HTTP status code $http_code after $attempt_seconds seconds" >&2 + fi else - echo "Error — Either HTTP response code for '$URL' is wrong - '$http_code' or exit code is not 0 - '$exit_code'. Waiting $interval seconds before the next attempt, $retries attempts left" - sleep 30 + attempt_seconds=$(($(date +%s) - attempt_start_time)) + echo "Package download failed in $attempt_seconds seconds" >&2 fi + + if [ $retries -eq 0 ]; then + total_seconds=$(($(date +%s) - download_start_time)) + echo "Package download failed after $total_seconds seconds" >&2 + exit 1 + fi + + echo "Waiting $interval seconds before retrying (retries left: $retries)..." >&2 + sleep $interval done - echo "Could not download $URL" - return 1 + echo "$download_path" } is_Arm64() { diff --git a/images/macos/scripts/helpers/xamarin-utils.sh b/images/macos/scripts/helpers/xamarin-utils.sh index d52c83a1a..83ddfb1bc 100644 --- a/images/macos/scripts/helpers/xamarin-utils.sh +++ b/images/macos/scripts/helpers/xamarin-utils.sh @@ -4,15 +4,6 @@ source ~/utils/utils.sh # Xamarin can clean their SDKs while updating to newer versions, # so we should be able to detect it during image generation -downloadAndInstallPKG() { - local PKG_URL=$1 - local PKG_NAME=${PKG_URL##*/} - - download_with_retries $PKG_URL - - echo "Installing $PKG_NAME..." - sudo installer -pkg "$TMPMOUNT/$PKG_NAME" -target / -} buildVSMacDownloadUrl() { echo "https://dl.xamarin.com/VsMac/VisualStudioForMac-${1}.dmg" @@ -41,7 +32,8 @@ installMono() { local MONO_FOLDER_NAME=$(echo $VERSION | cut -d. -f 1,2,3) local SHORT_VERSION=$(echo $VERSION | cut -d. -f 1,2) local PKG_URL=$(buildMonoDownloadUrl $VERSION) - downloadAndInstallPKG $PKG_URL + + sudo installer -pkg "$(download_with_retry "$PKG_URL")" -target / echo "Installing nunit3-console for Mono "$VERSION installNunitConsole $MONO_FOLDER_NAME @@ -59,7 +51,8 @@ installXamarinIOS() { echo "Installing Xamarin.iOS ${VERSION}..." local SHORT_VERSION=$(echo $VERSION | cut -d. -f 1,2) local PKG_URL=$(buildXamariniIOSDownloadUrl $VERSION) - downloadAndInstallPKG $PKG_URL + + sudo installer -pkg "$(download_with_retry "$PKG_URL")" -target / echo "Creating short symlink '${SHORT_VERSION}'" sudo ln -s ${IOS_VERSIONS_PATH}/${VERSION} ${IOS_VERSIONS_PATH}/${SHORT_VERSION} @@ -74,7 +67,8 @@ installXamarinMac() { echo "Installing Xamarin.Mac ${VERSION}..." local SHORT_VERSION=$(echo $VERSION | cut -d. -f 1,2) local PKG_URL=$(buildXamarinMacDownloadUrl $VERSION) - downloadAndInstallPKG $PKG_URL + + sudo installer -pkg "$(download_with_retry "$PKG_URL")" -target / echo "Creating short symlink '${SHORT_VERSION}'" sudo ln -s ${MAC_VERSIONS_PATH}/${VERSION} ${MAC_VERSIONS_PATH}/${SHORT_VERSION} @@ -89,7 +83,8 @@ installXamarinAndroid() { echo "Installing Xamarin.Android ${VERSION}..." local SHORT_VERSION=$(echo $VERSION | cut -d. -f 1,2) local PKG_URL=$(buildXamarinAndroidDownloadUrl $VERSION) - downloadAndInstallPKG $PKG_URL + + sudo installer -pkg "$(download_with_retry "$PKG_URL")" -target / if [ "$VERSION" == "9.4.1.0" ]; then # Fix symlinks for broken Xamarin.Android @@ -159,46 +154,27 @@ fixXamarinAndroidSymlinksInLibDir() { installNunitConsole() { local MONO_VERSION=$1 + local TMP_WRAPPER_PATH=$(mktemp) - cat < ${TMPMOUNT}/${NUNIT3_CONSOLE_BIN} + cat < "$TMP_WRAPPER_PATH" #!/bin/bash -e -o pipefail exec /Library/Frameworks/Mono.framework/Versions/${MONO_VERSION}/bin/mono --debug \$MONO_OPTIONS $NUNIT3_PATH/nunit3-console.exe "\$@" EOF - sudo chmod +x ${TMPMOUNT}/${NUNIT3_CONSOLE_BIN} - sudo mv ${TMPMOUNT}/${NUNIT3_CONSOLE_BIN} ${MONO_VERSIONS_PATH}/${MONO_VERSION}/Commands/${NUNIT3_CONSOLE_BIN} + sudo chmod +x "$TMP_WRAPPER_PATH" + sudo mv "$TMP_WRAPPER_PATH" "${MONO_VERSIONS_PATH}/${MONO_VERSION}/Commands/nunit3-console" } downloadNUnitConsole() { - echo "Downloading NUnit 3..." - local NUNIT3_LOCATION='https://github.com/nunit/nunit-console/releases/download/3.6.1/NUnit.Console-3.6.1.zip' - local NUNIT_PATH="/Library/Developer/nunit" - NUNIT3_PATH="$NUNIT_PATH/3.6.1" + echo "Downloading NUnit 3..." + local NUNIT3_VERSION='3.6.1' + local NUNIT3_LOCATION="https://github.com/nunit/nunit-console/releases/download/${NUNIT3_VERSION}/NUnit.Console-${NUNIT3_VERSION}.zip" + local NUNIT3_PATH="/Library/Developer/nunit/${NUNIT3_VERSION}" - pushd $TMPMOUNT + NUNIT3_ARCHIVE=$(download_with_retry $NUNIT3_LOCATION) - sudo mkdir -p $NUNIT3_PATH - download_with_retries $NUNIT3_LOCATION "." "nunit3.zip" - - echo "Installing NUnit 3..." - sudo unzip nunit3.zip -d $NUNIT3_PATH - NUNIT3_CONSOLE_BIN=nunit3-console - - popd -} - -installNuget() { - local MONO_VERSION=$1 - local NUGET_VERSION=$2 - local NUGET_URL="https://dist.nuget.org/win-x86-commandline/v${NUGET_VERSION}/nuget.exe" - echo "Installing nuget $NUGET_VERSION for Mono $MONO_VERSION" - cd ${MONO_VERSIONS_PATH}/${MONO_VERSION}/lib/mono/nuget - sudo mv nuget.exe nuget_old.exe - - pushd $TMPMOUNT - download_with_retries $NUGET_URL "." "nuget.exe" - sudo chmod a+x nuget.exe - sudo mv nuget.exe ${MONO_VERSIONS_PATH}/${MONO_VERSION}/lib/mono/nuget - popd + echo "Installing NUnit 3..." + sudo mkdir -p $NUNIT3_PATH + sudo unzip "$NUNIT3_ARCHIVE" -d $NUNIT3_PATH } createUWPShim() { @@ -236,4 +212,4 @@ deleteSymlink() { sudo rm -f ${IOS_VERSIONS_PATH}/${1} sudo rm -f ${MAC_VERSIONS_PATH}/${1} sudo rm -f ${ANDROID_VERSIONS_PATH}/${1} -} \ No newline at end of file +}