diff --git a/images/ubuntu/scripts/build/install-actions-cache.sh b/images/ubuntu/scripts/build/install-actions-cache.sh index 59312ba6c..416dab22a 100644 --- a/images/ubuntu/scripts/build/install-actions-cache.sh +++ b/images/ubuntu/scripts/build/install-actions-cache.sh @@ -18,8 +18,7 @@ setEtcEnvironmentVariable "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE" "${ACTION_ARCHIV # Download latest release from github.com/actions/action-versions and untar to /opt/actionarchivecache downloadUrl=$(get_github_package_download_url "actions/action-versions" "contains(\"action-versions.tar.gz\")") -echo "Downloading action-versions $downloadUrl" -download_with_retries "$downloadUrl" "/tmp" action-versions.tar.gz -tar -xzf /tmp/action-versions.tar.gz -C $ACTION_ARCHIVE_CACHE_DIR +archive_path=$(download_with_retry "$downloadUrl") +tar -xzf "$archive_path" -C $ACTION_ARCHIVE_CACHE_DIR invoke_tests "ActionArchiveCache" diff --git a/images/ubuntu/scripts/build/install-aliyun-cli.sh b/images/ubuntu/scripts/build/install-aliyun-cli.sh index 95bcc2f48..d3d8813bf 100644 --- a/images/ubuntu/scripts/build/install-aliyun-cli.sh +++ b/images/ubuntu/scripts/build/install-aliyun-cli.sh @@ -19,8 +19,7 @@ else hash_url="https://github.com/aliyun/aliyun-cli/releases/latest/download/SHASUMS256.txt" fi -package_name="aliyun-cli-linux-amd64.tgz" -download_with_retries "$download_url" "/tmp" "$package_name" +archive_path=$(download_with_retry "$download_url") # Supply chain security - Alibaba Cloud CLI if isUbuntu20; then @@ -29,9 +28,9 @@ else external_hash=$(get_hash_from_remote_file "$hash_url" "aliyun-cli-linux" "amd64.tgz") fi -use_checksum_comparison "/tmp/$package_name" "$external_hash" +use_checksum_comparison "$archive_path" "$external_hash" -tar xzf "/tmp/$package_name" +tar xzf "$archive_path" mv aliyun /usr/local/bin invoke_tests "CLI.Tools" "Aliyun CLI" diff --git a/images/ubuntu/scripts/build/install-android-sdk.sh b/images/ubuntu/scripts/build/install-android-sdk.sh index 7521222e2..3b614eed7 100644 --- a/images/ubuntu/scripts/build/install-android-sdk.sh +++ b/images/ubuntu/scripts/build/install-android-sdk.sh @@ -43,18 +43,16 @@ setEtcEnvironmentVariable "ANDROID_HOME" "${ANDROID_SDK_ROOT}" # Create android sdk directory mkdir -p ${ANDROID_SDK_ROOT} -cmdlineTools="android-cmdline-tools.zip" - # Download the latest command line tools so that we can accept all of the licenses. # See https://developer.android.com/studio/#command-tools cmdlineToolsVersion=$(get_toolset_value '.android."cmdline-tools"') if [[ $cmdlineToolsVersion == "latest" ]]; then - repositoryXmlUrl="https://dl.google.com/android/repository/repository2-1.xml" - download_with_retries $repositoryXmlUrl "/tmp" "repository2-1.xml" + repository_xml_url="https://dl.google.com/android/repository/repository2-1.xml" + repository_xml_file=$(download_with_retry "$repository_xml_url") cmdlineToolsVersion=$( yq -p=xml \ '.sdk-repository.remotePackage[] | select(."+@path" == "cmdline-tools;latest" and .channelRef."+@ref" == "channel-0").archives.archive[].complete.url | select(contains("commandlinetools-linux"))' \ - /tmp/repository2-1.xml + "${repository_xml_file}" ) if [[ -z $cmdlineToolsVersion ]]; then @@ -63,12 +61,10 @@ if [[ $cmdlineToolsVersion == "latest" ]]; then fi fi -download_with_retries "https://dl.google.com/android/repository/${cmdlineToolsVersion}" "." $cmdlineTools - -unzip -qq $cmdlineTools -d ${ANDROID_SDK_ROOT}/cmdline-tools +archive_path=$(download_with_retry "https://dl.google.com/android/repository/${cmdlineToolsVersion}") +unzip -qq "$archive_path" -d ${ANDROID_SDK_ROOT}/cmdline-tools # Command line tools need to be placed in ${ANDROID_SDK_ROOT}/sdk/cmdline-tools/latest to determine SDK root mv ${ANDROID_SDK_ROOT}/cmdline-tools/cmdline-tools ${ANDROID_SDK_ROOT}/cmdline-tools/latest -rm -f $cmdlineTools # Check sdk manager installation ${SDKMANAGER} --list 1>/dev/null diff --git a/images/ubuntu/scripts/build/install-aws-tools.sh b/images/ubuntu/scripts/build/install-aws-tools.sh index d5ee0576d..be2980538 100644 --- a/images/ubuntu/scripts/build/install-aws-tools.sh +++ b/images/ubuntu/scripts/build/install-aws-tools.sh @@ -9,23 +9,23 @@ source $HELPER_SCRIPTS/os.sh source $HELPER_SCRIPTS/install.sh -download_with_retries "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" "/tmp" "awscliv2.zip" -unzip -qq /tmp/awscliv2.zip -d /tmp +awscliv2_archive_path=$(download_with_retry "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip") +unzip -qq "$awscliv2_archive_path" -d /tmp /tmp/aws/install -i /usr/local/aws-cli -b /usr/local/bin -download_with_retries "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" "/tmp" "session-manager-plugin.deb" -apt install /tmp/session-manager-plugin.deb +smplugin_deb_path=$(download_with_retry "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb") +apt install "$smplugin_deb_path" # Download the latest aws sam cli release -aws_sam_cli_zip_name="aws-sam-cli-linux-x86_64.zip" -download_with_retries "https://github.com/aws/aws-sam-cli/releases/latest/download/${aws_sam_cli_zip_name}" "/tmp" $aws_sam_cli_zip_name +aws_sam_cli_archive_name="aws-sam-cli-linux-x86_64.zip" +aws_sam_cli_archive_path=$(download_with_retry "https://github.com/aws/aws-sam-cli/releases/latest/download/${aws_sam_cli_archive_name}") # Supply chain security - AWS SAM CLI -aws_sam_cli_hash=$(get_github_package_hash "aws" "aws-sam-cli" "${aws_sam_cli_zip_name}.. ") -use_checksum_comparison "/tmp/${aws_sam_cli_zip_name}" "$aws_sam_cli_hash" +aws_sam_cli_hash=$(get_github_package_hash "aws" "aws-sam-cli" "${aws_sam_cli_archive_name}.. ") +use_checksum_comparison "$aws_sam_cli_archive_path" "$aws_sam_cli_hash" # Install the latest aws sam cli release -unzip /tmp/${aws_sam_cli_zip_name} -d /tmp +unzip "$aws_sam_cli_archive_path" -d /tmp /tmp/install invoke_tests "CLI.Tools" "AWS" diff --git a/images/ubuntu/scripts/build/install-azcopy.sh b/images/ubuntu/scripts/build/install-azcopy.sh index f06df251d..a93d663d1 100644 --- a/images/ubuntu/scripts/build/install-azcopy.sh +++ b/images/ubuntu/scripts/build/install-azcopy.sh @@ -7,10 +7,9 @@ source $HELPER_SCRIPTS/install.sh # Install AzCopy10 -download_with_retries "https://aka.ms/downloadazcopy-v10-linux" "/tmp" "azcopy.tar.gz" -tar xzf /tmp/azcopy.tar.gz --strip-components=1 -C /tmp -mv /tmp/azcopy /usr/local/bin/azcopy -chmod +x /usr/local/bin/azcopy +archive_path=$(download_with_retry "https://aka.ms/downloadazcopy-v10-linux") +tar xzf "$archive_path" --strip-components=1 -C /tmp +install /tmp/azcopy /usr/local/bin/azcopy # Create azcopy 10 alias for backward compatibility ln -sf /usr/local/bin/azcopy /usr/local/bin/azcopy10 diff --git a/images/ubuntu/scripts/build/install-bicep.sh b/images/ubuntu/scripts/build/install-bicep.sh index a568ba73d..840b95310 100644 --- a/images/ubuntu/scripts/build/install-bicep.sh +++ b/images/ubuntu/scripts/build/install-bicep.sh @@ -7,10 +7,8 @@ source $HELPER_SCRIPTS/install.sh # Install Bicep CLI -download_with_retries "https://github.com/Azure/bicep/releases/latest/download/bicep-linux-x64" "." "bicep.bin" +bicep_binary_path=$(download_with_retry "https://github.com/Azure/bicep/releases/latest/download/bicep-linux-x64") # Mark it as executable -chmod +x ./bicep.bin -# Add bicep to PATH (requires admin) -sudo mv ./bicep.bin /usr/local/bin/bicep +install "$bicep_binary_path" /usr/local/bin/bicep invoke_tests "Tools" "Bicep" diff --git a/images/ubuntu/scripts/build/install-codeql-bundle.sh b/images/ubuntu/scripts/build/install-codeql-bundle.sh index 9cc01bed8..c9dd18120 100644 --- a/images/ubuntu/scripts/build/install-codeql-bundle.sh +++ b/images/ubuntu/scripts/build/install-codeql-bundle.sh @@ -14,8 +14,7 @@ bundle_tag_name="codeql-bundle-v$bundle_version" echo "Downloading CodeQL bundle $bundle_version..." # Note that this is the all-platforms CodeQL bundle, to support scenarios where customers run # different operating systems within containers. -download_with_retries "https://github.com/github/codeql-action/releases/download/$bundle_tag_name/codeql-bundle.tar.gz" "/tmp" "codeql-bundle.tar.gz" -codeql_archive="/tmp/codeql-bundle.tar.gz" +codeql_archive=$(download_with_retry "https://github.com/github/codeql-action/releases/download/$bundle_tag_name/codeql-bundle.tar.gz") codeql_toolcache_path="$AGENT_TOOLSDIRECTORY/CodeQL/$bundle_version/x64" mkdir -p "$codeql_toolcache_path" diff --git a/images/ubuntu/scripts/build/install-docker.sh b/images/ubuntu/scripts/build/install-docker.sh index 817cc2476..bbf77048f 100644 --- a/images/ubuntu/scripts/build/install-docker.sh +++ b/images/ubuntu/scripts/build/install-docker.sh @@ -64,15 +64,14 @@ else fi # Download amazon-ecr-credential-helper -aws_helper="docker-credential-ecr-login" aws_latest_release_url="https://api.github.com/repos/awslabs/amazon-ecr-credential-helper/releases/latest" aws_helper_url=$(curl "${authString[@]}" -fsSL "${aws_latest_release_url}" | jq -r '.body' | awk -F'[()]' '/linux-amd64/ {print $2}') -download_with_retries "${aws_helper_url}" "/tmp" "${aws_helper}" +aws_helper_binary_path=$(download_with_retry "$aws_helper_url") # Supply chain security - amazon-ecr-credential-helper -aws_helper_external_hash=$(get_hash_from_remote_file "${aws_helper_url}.sha256" "${aws_helper}") -use_checksum_comparison "/tmp/${aws_helper}" "${aws_helper_external_hash}" +aws_helper_external_hash=$(get_hash_from_remote_file "${aws_helper_url}.sha256" "docker-credential-ecr-login") +use_checksum_comparison "$aws_helper_binary_path" "$aws_helper_external_hash" # Install amazon-ecr-credential-helper -install "/tmp/${aws_helper}" "/usr/bin/${aws_helper}" +install "$aws_helper_binary_path" "/usr/bin/docker-credential-ecr-login" # Cleanup custom repositories rm $gpg_key diff --git a/images/ubuntu/scripts/build/install-dotnetcore-sdk.sh b/images/ubuntu/scripts/build/install-dotnetcore-sdk.sh index 30f38d9b3..3eef7af1a 100644 --- a/images/ubuntu/scripts/build/install-dotnetcore-sdk.sh +++ b/images/ubuntu/scripts/build/install-dotnetcore-sdk.sh @@ -47,15 +47,13 @@ apt-get update sdks=() for version in ${DOTNET_VERSIONS[@]}; do release_url="https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/${version}/releases.json" - download_with_retries "${release_url}" "." "${version}.json" - releases=$(cat "./${version}.json") + releases=$(cat "$(download_with_retry "$release_url")") if [[ $version == "6.0" ]]; then sdks=("${sdks[@]}" $(echo "${releases}" | jq -r 'first(.releases[].sdks[]?.version | select(contains("preview") or contains("rc") | not))')) else sdks=("${sdks[@]}" $(echo "${releases}" | jq -r '.releases[].sdk.version | select(contains("preview") or contains("rc") | not)')) sdks=("${sdks[@]}" $(echo "${releases}" | jq -r '.releases[].sdks[]?.version | select(contains("preview") or contains("rc") | not)')) fi - rm ./${version}.json done sortedSdks=$(echo ${sdks[@]} | tr ' ' '\n' | sort -r | uniq -w 5) @@ -73,12 +71,12 @@ extract_dotnet_sdk() { } # Download/install additional SDKs in parallel -export -f download_with_retries +export -f download_with_retry export -f extract_dotnet_sdk parallel --jobs 0 --halt soon,fail=1 \ 'url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/{}/dotnet-sdk-{}-linux-x64.tar.gz"; \ - download_with_retries $url' ::: "${sortedSdks[@]}" + download_with_retry $url' ::: "${sortedSdks[@]}" find . -name "*.tar.gz" | parallel --halt soon,fail=1 'extract_dotnet_sdk {}' diff --git a/images/ubuntu/scripts/build/install-erlang.sh b/images/ubuntu/scripts/build/install-erlang.sh index a0e730f8f..e78439cae 100644 --- a/images/ubuntu/scripts/build/install-erlang.sh +++ b/images/ubuntu/scripts/build/install-erlang.sh @@ -19,8 +19,8 @@ apt-get install --no-install-recommends esl-erlang # Install rebar3 rebar3_url="https://github.com/erlang/rebar3/releases/latest/download/rebar3" -download_with_retries $rebar3_url "/usr/local/bin" "rebar3" -chmod +x /usr/local/bin/rebar3 +rebar3_binary_path=$(download_with_retry "$rebar3_url") +install "$rebar3_binary_path" /usr/local/bin/rebar3 # Clean up source list rm $source_list diff --git a/images/ubuntu/scripts/build/install-firefox.sh b/images/ubuntu/scripts/build/install-firefox.sh index 628abde2e..53714ca1c 100644 --- a/images/ubuntu/scripts/build/install-firefox.sh +++ b/images/ubuntu/scripts/build/install-firefox.sh @@ -33,14 +33,14 @@ echo "mozillateam $repo_url" >> $HELPER_SCRIPTS/apt-sources.txt echo 'pref("intl.locale.requested","en_US");' >> "/usr/lib/firefox/browser/defaults/preferences/syspref.js" # Download and unpack latest release of geckodriver -downloadUrl=$(get_github_package_download_url "mozilla/geckodriver" "test(\"linux64.tar.gz$\")") -download_with_retries "$downloadUrl" "/tmp" geckodriver.tar.gz +download_url=$(get_github_package_download_url "mozilla/geckodriver" "test(\"linux64.tar.gz$\")") +driver_archive_path=$(download_with_retry "$download_url") GECKODRIVER_DIR="/usr/local/share/gecko_driver" GECKODRIVER_BIN="$GECKODRIVER_DIR/geckodriver" mkdir -p $GECKODRIVER_DIR -tar -xzf /tmp/geckodriver.tar.gz -C $GECKODRIVER_DIR +tar -xzf "$driver_archive_path" -C $GECKODRIVER_DIR chmod +x $GECKODRIVER_BIN ln -s "$GECKODRIVER_BIN" /usr/bin/ diff --git a/images/ubuntu/scripts/build/install-github-cli.sh b/images/ubuntu/scripts/build/install-github-cli.sh index 65e3cddcc..6e43aed4a 100644 --- a/images/ubuntu/scripts/build/install-github-cli.sh +++ b/images/ubuntu/scripts/build/install-github-cli.sh @@ -10,13 +10,13 @@ source $HELPER_SCRIPTS/install.sh # Download GitHub CLI -URL=$(get_github_package_download_url "cli/cli" "contains(\"linux\") and contains(\"amd64\") and contains(\".deb\")") -download_with_retries "${URL}" "/tmp" "gh_cli_linux_amd64.deb" +gh_cli_url=$(get_github_package_download_url "cli/cli" "contains(\"linux\") and contains(\"amd64\") and contains(\".deb\")") +gh_cli_deb_path=$(download_with_retry "$gh_cli_url") # Supply chain security - GitHub CLI hash_url=$(get_github_package_download_url "cli/cli" "contains(\"checksums.txt\")") -external_hash=$(get_hash_from_remote_file "${hash_url}" "linux_amd64.deb") -use_checksum_comparison "/tmp/gh_cli_linux_amd64.deb" "${external_hash}" +external_hash=$(get_hash_from_remote_file "$hash_url" "linux_amd64.deb") +use_checksum_comparison "$gh_cli_deb_path" "$external_hash" # Install GitHub CLI -apt install /tmp/gh_cli_linux_amd64.deb +apt install "$gh_cli_deb_path" invoke_tests "CLI.Tools" "GitHub CLI" diff --git a/images/ubuntu/scripts/build/install-google-chrome.sh b/images/ubuntu/scripts/build/install-google-chrome.sh index 4608bc6b7..88b920208 100644 --- a/images/ubuntu/scripts/build/install-google-chrome.sh +++ b/images/ubuntu/scripts/build/install-google-chrome.sh @@ -34,9 +34,8 @@ function GetChromiumRevision { # Download and install Google Chrome CHROME_DEB_URL="https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb" -CHROME_DEB_NAME="google-chrome-stable_current_amd64.deb" -download_with_retries $CHROME_DEB_URL "/tmp" "${CHROME_DEB_NAME}" -apt install "/tmp/${CHROME_DEB_NAME}" -f +CHROME_DEB_PATH=$(download_with_retry "$CHROME_DEB_URL") +apt install "$CHROME_DEB_PATH" -f setEtcEnvironmentVariable "CHROME_BIN" "/usr/bin/google-chrome" # Remove Google Chrome repo @@ -55,13 +54,12 @@ CHROME_VERSIONS_JSON=$(curl -fsSL "${CHROME_VERSIONS_URL}") # Download and unpack the latest release of chromedriver CHROMEDRIVER_VERSION=$(echo "${CHROME_VERSIONS_JSON}" | jq -r '.builds["'"$CHROME_VERSION"'"].version') CHROMEDRIVER_URL=$(echo "${CHROME_VERSIONS_JSON}" | jq -r '.builds["'"$CHROME_VERSION"'"].downloads.chromedriver[] | select(.platform=="'"${CHROME_PLATFORM}"'").url') -CHROMEDRIVER_ARCHIVE="chromedriver_linux64.zip" CHROMEDRIVER_DIR="/usr/local/share/chromedriver-linux64" CHROMEDRIVER_BIN="$CHROMEDRIVER_DIR/chromedriver" echo "Installing chromedriver version $CHROMEDRIVER_VERSION" -download_with_retries $CHROMEDRIVER_URL "/tmp" $CHROMEDRIVER_ARCHIVE -unzip -qq /tmp/$CHROMEDRIVER_ARCHIVE -d /usr/local/share +driver_archive_path=$(download_with_retry "$CHROMEDRIVER_URL") +unzip -qq "$driver_archive_path" -d /usr/local/share chmod +x $CHROMEDRIVER_BIN ln -s "$CHROMEDRIVER_BIN" /usr/bin/ @@ -71,14 +69,13 @@ setEtcEnvironmentVariable "CHROMEWEBDRIVER" "${CHROMEDRIVER_DIR}" CHROME_REVISION=$(echo "${CHROME_VERSIONS_JSON}" | jq -r '.builds["'"$CHROME_VERSION"'"].revision') CHROMIUM_REVISION=$(GetChromiumRevision $CHROME_REVISION) CHROMIUM_URL="https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${CHROMIUM_REVISION}%2Fchrome-linux.zip?alt=media" -CHROMIUM_ARCHIVE="${CHROMIUM_REVISION}-chromium-linux.zip" CHROMIUM_DIR="/usr/local/share/chromium" CHROMIUM_BIN="${CHROMIUM_DIR}/chrome-linux/chrome" echo "Installing chromium revision $CHROMIUM_REVISION" -download_with_retries $CHROMIUM_URL "/tmp" $CHROMIUM_ARCHIVE +CHROMIUM_ARCHIVE_PATH=$(download_with_retry "$CHROMIUM_URL") mkdir $CHROMIUM_DIR -unzip -qq /tmp/$CHROMIUM_ARCHIVE -d $CHROMIUM_DIR +unzip -qq "$CHROMIUM_ARCHIVE_PATH" -d $CHROMIUM_DIR ln -s $CHROMIUM_BIN /usr/bin/chromium ln -s $CHROMIUM_BIN /usr/bin/chromium-browser diff --git a/images/ubuntu/scripts/build/install-java-tools.sh b/images/ubuntu/scripts/build/install-java-tools.sh index 65360fd7c..bee2b5fd0 100644 --- a/images/ubuntu/scripts/build/install-java-tools.sh +++ b/images/ubuntu/scripts/build/install-java-tools.sh @@ -98,8 +98,8 @@ setEtcEnvironmentVariable "ANT_HOME" "/usr/share/ant" # Install Maven mavenVersion=$(get_toolset_value '.java.maven') mavenDownloadUrl="https://dlcdn.apache.org/maven/maven-3/${mavenVersion}/binaries/apache-maven-${mavenVersion}-bin.zip" -download_with_retries ${mavenDownloadUrl} "/tmp" "maven.zip" -unzip -qq -d /usr/share /tmp/maven.zip +maven_archive_path=$(download_with_retry "$mavenDownloadUrl") +unzip -qq -d /usr/share "$maven_archive_path" ln -s /usr/share/apache-maven-${mavenVersion}/bin/mvn /usr/bin/mvn # Install Gradle @@ -110,8 +110,8 @@ gradleLatestVersion=$(echo ${gradleJson} | jq -r '.[] | select(.version | contai gradleDownloadUrl=$(echo ${gradleJson} | jq -r ".[] | select(.version==\"$gradleLatestVersion\") | .downloadUrl") echo "gradleUrl=${gradleDownloadUrl}" echo "gradleVersion=${gradleLatestVersion}" -download_with_retries ${gradleDownloadUrl} "/tmp" "gradleLatest.zip" -unzip -qq -d /usr/share /tmp/gradleLatest.zip +gradle_archive_path=$(download_with_retry "$gradleDownloadUrl") +unzip -qq -d /usr/share "$gradle_archive_path" ln -s /usr/share/gradle-"${gradleLatestVersion}"/bin/gradle /usr/bin/gradle gradle_home_dir=$(find /usr/share -depth -maxdepth 1 -name "gradle*") setEtcEnvironmentVariable "GRADLE_HOME" "${gradle_home_dir}" diff --git a/images/ubuntu/scripts/build/install-julia.sh b/images/ubuntu/scripts/build/install-julia.sh index 4c491d2cd..46c4aaf22 100644 --- a/images/ubuntu/scripts/build/install-julia.sh +++ b/images/ubuntu/scripts/build/install-julia.sh @@ -13,15 +13,12 @@ julia_version=$(echo $json | jq -r '.[].files[] | select(.triplet=="x86_64-linux # download julia archive julia_tar_url=$(echo $json | jq -r ".[].files[].url | select(endswith(\"julia-${julia_version}-linux-x86_64.tar.gz\"))") -julia_tar_name="julia-${julia_version}-linux-x86_64.tar.gz" -download_with_retries $julia_tar_url "/tmp" "${julia_tar_name}" +julia_archive_path=$(download_with_retry "$julia_tar_url") # extract files and make symlink -julia_tar_tmp="/tmp/${julia_tar_name}" julia_installation_path="/usr/local/julia${julia_version}" mkdir -p "${julia_installation_path}" -tar -C "${julia_installation_path}" -xzf "${julia_tar_tmp}" --strip-components=1 +tar -C "${julia_installation_path}" -xzf "$julia_archive_path" --strip-components=1 ln -s "${julia_installation_path}/bin/julia" /usr/bin/julia -rm "${julia_tar_tmp}" invoke_tests "Tools" "Julia" diff --git a/images/ubuntu/scripts/build/install-kotlin.sh b/images/ubuntu/scripts/build/install-kotlin.sh index 1995a66e1..a8c2bd273 100644 --- a/images/ubuntu/scripts/build/install-kotlin.sh +++ b/images/ubuntu/scripts/build/install-kotlin.sh @@ -8,15 +8,14 @@ source $HELPER_SCRIPTS/install.sh KOTLIN_ROOT="/usr/share" -kotlin_zip_name="kotlin-compiler.zip" download_url=$(get_github_package_download_url "JetBrains/kotlin" "contains(\"kotlin-compiler\")") -download_with_retries "$download_url" "/tmp" "$kotlin_zip_name" +archive_path=$(download_with_retry "$download_url") # Supply chain security - Kotlin kotlin_hash=$(get_github_package_hash "JetBrains" "kotlin" "kotlin-compiler-.*\.zip" "" "latest" "false" "|" 3) -use_checksum_comparison "/tmp/${kotlin_zip_name}" "$kotlin_hash" +use_checksum_comparison "$archive_path" "$kotlin_hash" -unzip -qq /tmp/${kotlin_zip_name} -d $KOTLIN_ROOT +unzip -qq "$archive_path" -d $KOTLIN_ROOT rm $KOTLIN_ROOT/kotlinc/bin/*.bat ln -sf $KOTLIN_ROOT/kotlinc/bin/* /usr/bin diff --git a/images/ubuntu/scripts/build/install-microsoft-edge.sh b/images/ubuntu/scripts/build/install-microsoft-edge.sh index 2103102b7..a52f4cb27 100644 --- a/images/ubuntu/scripts/build/install-microsoft-edge.sh +++ b/images/ubuntu/scripts/build/install-microsoft-edge.sh @@ -40,9 +40,9 @@ EDGE_DRIVER_VERSION_URL="https://msedgedriver.azureedge.net/LATEST_RELEASE_${EDG EDGE_DRIVER_LATEST_VERSION=$(curl -fsSL "$EDGE_DRIVER_VERSION_URL" | iconv -f utf-16 -t utf-8 | tr -d '\r') EDGEDRIVER_URL="https://msedgedriver.azureedge.net/${EDGE_DRIVER_LATEST_VERSION}/edgedriver_linux64.zip" -download_with_retries $EDGEDRIVER_URL "/tmp" "edgedriver_linux64.zip" +EDGEDRIVER_ARCHIVE_PATH=$(download_with_retry "$EDGEDRIVER_URL") -unzip -qq /tmp/edgedriver_linux64.zip -d $EDGEDRIVER_DIR +unzip -qq "$EDGEDRIVER_ARCHIVE_PATH" -d "$EDGEDRIVER_DIR" chmod +x $EDGEDRIVER_BIN ln -s $EDGEDRIVER_BIN /usr/bin diff --git a/images/ubuntu/scripts/build/install-oc-cli.sh b/images/ubuntu/scripts/build/install-oc-cli.sh index 36ec84794..f1094ff6a 100644 --- a/images/ubuntu/scripts/build/install-oc-cli.sh +++ b/images/ubuntu/scripts/build/install-oc-cli.sh @@ -7,9 +7,8 @@ source $HELPER_SCRIPTS/install.sh # Install the oc CLI -DOWNLOAD_URL="https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/openshift-client-linux.tar.gz" -PACKAGE_TAR_NAME="oc.tar.gz" -download_with_retries $DOWNLOAD_URL "/tmp" $PACKAGE_TAR_NAME -tar xzf "/tmp/$PACKAGE_TAR_NAME" -C "/usr/local/bin" oc +download_url="https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/openshift-client-linux.tar.gz" +archive_path=$(download_with_retry "$download_url") +tar xzf "$archive_path" -C "/usr/local/bin" oc invoke_tests "CLI.Tools" "OC CLI" diff --git a/images/ubuntu/scripts/build/install-oras-cli.sh b/images/ubuntu/scripts/build/install-oras-cli.sh index f8481a91d..352a94b73 100644 --- a/images/ubuntu/scripts/build/install-oras-cli.sh +++ b/images/ubuntu/scripts/build/install-oras-cli.sh @@ -8,16 +8,17 @@ source $HELPER_SCRIPTS/install.sh # Determine latest ORAS CLI version -URL=$(get_github_package_download_url "oras-project/oras" "endswith(\"linux_amd64.tar.gz\")") -archive_name=$(basename "${URL}") +download_url=$(get_github_package_download_url "oras-project/oras" "endswith(\"linux_amd64.tar.gz\")") # Download ORAS CLI -download_with_retries "${URL}" "/tmp" "${archive_name}" +archive_path=$(download_with_retry "$download_url") + # Supply chain security - ORAS CLI hash_url=$(get_github_package_download_url "oras-project/oras" "contains(\"checksums.txt\")") external_hash=$(get_hash_from_remote_file "${hash_url}" "linux_amd64.tar.gz") -use_checksum_comparison "/tmp/${archive_name}" "${external_hash}" +use_checksum_comparison "$archive_path" "${external_hash}" + # Unzip ORAS CLI -tar xzf "/tmp/${archive_name}" -C /usr/local/bin oras +tar xzf "$archive_path" -C /usr/local/bin oras invoke_tests "CLI.Tools" "Oras CLI" diff --git a/images/ubuntu/scripts/build/install-packer.sh b/images/ubuntu/scripts/build/install-packer.sh index cdcdc5673..5fb0bd8b5 100644 --- a/images/ubuntu/scripts/build/install-packer.sh +++ b/images/ubuntu/scripts/build/install-packer.sh @@ -7,10 +7,8 @@ source $HELPER_SCRIPTS/install.sh # Install Packer -URL=$(curl -fsSL https://api.releases.hashicorp.com/v1/releases/packer/latest | jq -r '.builds[] | select((.arch=="amd64") and (.os=="linux")).url') -ZIP_NAME="packer_linux_amd64.zip" -download_with_retries "${URL}" "/tmp" "${ZIP_NAME}" -unzip -qq "/tmp/${ZIP_NAME}" -d /usr/local/bin -rm -f "/tmp/${ZIP_NAME}" +download_url=$(curl -fsSL https://api.releases.hashicorp.com/v1/releases/packer/latest | jq -r '.builds[] | select((.arch=="amd64") and (.os=="linux")).url') +archive_path=$(download_with_retry "$download_url") +unzip -qq "$archive_path" -d /usr/local/bin invoke_tests "Tools" "Packer" diff --git a/images/ubuntu/scripts/build/install-phantomjs.sh b/images/ubuntu/scripts/build/install-phantomjs.sh index 721cd6854..2ed14a048 100644 --- a/images/ubuntu/scripts/build/install-phantomjs.sh +++ b/images/ubuntu/scripts/build/install-phantomjs.sh @@ -7,13 +7,20 @@ # Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh -# Install PhantomJS +# Install required dependencies apt-get install -y chrpath libssl-dev libxft-dev libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev -phantom_js=phantomjs-2.1.1-linux-x86_64 -download_with_retries https://bitbucket.org/ariya/phantomjs/downloads/$phantom_js.tar.bz2 "/tmp" -phantom_js_hash="86dd9a4bf4aee45f1a84c9f61cf1947c1d6dce9b9e8d2a907105da7852460d2f" -use_checksum_comparison "/tmp/${phantom_js}.tar.bz2" "${phantom_js_hash}" -tar xjf /tmp/$phantom_js.tar.bz2 -C /usr/local/share -ln -sf /usr/local/share/$phantom_js/bin/phantomjs /usr/local/bin + +# Define the version and hash of PhantomJS to be installed +dir_name=phantomjs-2.1.1-linux-x86_64 +archive_hash="86dd9a4bf4aee45f1a84c9f61cf1947c1d6dce9b9e8d2a907105da7852460d2f" + +# Download the archive and verify its integrity using checksum comparison +download_url="https://bitbucket.org/ariya/phantomjs/downloads/$dir_name.tar.bz2" +archive_path=$(download_with_retry "$download_url") +use_checksum_comparison "$archive_path" "$archive_hash" + +# Extract the archive and create a symbolic link to the executable +tar xjf "$archive_path" -C /usr/local/share +ln -sf /usr/local/share/$dir_name/bin/phantomjs /usr/local/bin invoke_tests "Tools" "Phantomjs" diff --git a/images/ubuntu/scripts/build/install-pulumi.sh b/images/ubuntu/scripts/build/install-pulumi.sh index 19fce9a49..33d4d9533 100644 --- a/images/ubuntu/scripts/build/install-pulumi.sh +++ b/images/ubuntu/scripts/build/install-pulumi.sh @@ -10,12 +10,14 @@ source $HELPER_SCRIPTS/install.sh # Dowload Pulumi version=$(curl -fsSL "https://www.pulumi.com/latest-version") -URL="https://get.pulumi.com/releases/sdk/pulumi-v${version}-linux-x64.tar.gz" -download_with_retries "${URL}" "/tmp" "pulumi-v${version}.tar.gz" +download_url="https://get.pulumi.com/releases/sdk/pulumi-v${version}-linux-x64.tar.gz" +archive_path=$(download_with_retry "$download_url") + # Supply chain security - Pulumi external_hash=$(get_hash_from_remote_file "https://github.com/pulumi/pulumi/releases/download/v${version}/SHA512SUMS" "linux-x64.tar.gz") -use_checksum_comparison "/tmp/pulumi-v${version}.tar.gz" "${external_hash}" "512" +use_checksum_comparison "$archive_path" "$external_hash" "512" + # Unzipping Pulumi -tar --strip=1 -xf "/tmp/pulumi-v${version}.tar.gz" -C /usr/local/bin +tar --strip=1 -xf "$archive_path" -C /usr/local/bin invoke_tests "Tools" "Pulumi" diff --git a/images/ubuntu/scripts/build/install-pypy.sh b/images/ubuntu/scripts/build/install-pypy.sh index e4797c0e2..51f09cbc5 100644 --- a/images/ubuntu/scripts/build/install-pypy.sh +++ b/images/ubuntu/scripts/build/install-pypy.sh @@ -12,16 +12,16 @@ function InstallPyPy { PACKAGE_URL=$1 - PACKAGE_TAR_NAME=$(echo $PACKAGE_URL | awk -F/ '{print $NF}') - echo "Downloading tar archive '$PACKAGE_TAR_NAME'" - PACKAGE_TAR_TEMP_PATH="/tmp/$PACKAGE_TAR_NAME" - download_with_retries $PACKAGE_URL "/tmp" $PACKAGE_TAR_NAME + PACKAGE_TAR_NAME=$(echo "$PACKAGE_URL" | awk -F/ '{print $NF}') + PACKAGE_NAME=${PACKAGE_TAR_NAME/.tar.bz2/} - echo "Expand '$PACKAGE_TAR_NAME' to the /tmp folder" - tar xf $PACKAGE_TAR_TEMP_PATH -C /tmp + echo "Downloading tar archive '$PACKAGE_NAME'" + PACKAGE_TAR_TEMP_PATH=$(download_with_retry $PACKAGE_URL) + + echo "Expand '$PACKAGE_NAME' to the /tmp folder" + tar xf "$PACKAGE_TAR_TEMP_PATH" -C /tmp # Get Python version - PACKAGE_NAME=${PACKAGE_TAR_NAME/.tar.bz2/} MAJOR_VERSION=$(echo ${PACKAGE_NAME/pypy/} | cut -d. -f1) PYTHON_MAJOR="python$MAJOR_VERSION" diff --git a/images/ubuntu/scripts/build/install-ruby.sh b/images/ubuntu/scripts/build/install-ruby.sh index de5de8061..5241a73c4 100644 --- a/images/ubuntu/scripts/build/install-ruby.sh +++ b/images/ubuntu/scripts/build/install-ruby.sh @@ -42,10 +42,10 @@ for TOOLSET_VERSION in ${TOOLSET_VERSIONS[@]}; do echo "Downloading tar archive $PACKAGE_TAR_NAME" DOWNLOAD_URL="https://github.com/ruby/ruby-builder/releases/download/toolcache/${PACKAGE_TAR_NAME}" - download_with_retries $DOWNLOAD_URL "/tmp" $PACKAGE_TAR_NAME + PACKAGE_ARCHIVE_PATH=$(download_with_retry "$DOWNLOAD_URL") echo "Expand '$PACKAGE_TAR_NAME' to the '$RUBY_VERSION_PATH' folder" - tar xf "/tmp/$PACKAGE_TAR_NAME" -C $RUBY_VERSION_PATH + tar xf "$PACKAGE_ARCHIVE_PATH" -C $RUBY_VERSION_PATH COMPLETE_FILE_PATH="$RUBY_VERSION_PATH/x64.complete" if [ ! -f $COMPLETE_FILE_PATH ]; then diff --git a/images/ubuntu/scripts/build/install-runner-package.sh b/images/ubuntu/scripts/build/install-runner-package.sh index 569a4a61c..967c0466e 100644 --- a/images/ubuntu/scripts/build/install-runner-package.sh +++ b/images/ubuntu/scripts/build/install-runner-package.sh @@ -7,12 +7,9 @@ # Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh -DOWNLOAD_URL=$(get_github_package_download_url "actions/runner" 'test("actions-runner-linux-x64-[0-9]+\\.[0-9]{3}\\.[0-9]+\\.tar\\.gz")') +download_url=$(get_github_package_download_url "actions/runner" 'test("actions-runner-linux-x64-[0-9]+\\.[0-9]{3}\\.[0-9]+\\.tar\\.gz")') +archive_name="${download_url##*/}" +archive_path=$(download_with_retry "$download_url") -FILE_NAME="${DOWNLOAD_URL##*/}" - -sudo mkdir -p /opt/runner-cache - -download_with_retries "${DOWNLOAD_URL}" "/tmp" "${FILE_NAME}" - -sudo mv /tmp/$FILE_NAME /opt/runner-cache/$FILE_NAME +mkdir -p /opt/runner-cache +mv "$archive_path" "/opt/runner-cache/$archive_name" diff --git a/images/ubuntu/scripts/build/install-sbt.sh b/images/ubuntu/scripts/build/install-sbt.sh index ff90abced..ae7610f2d 100644 --- a/images/ubuntu/scripts/build/install-sbt.sh +++ b/images/ubuntu/scripts/build/install-sbt.sh @@ -7,9 +7,9 @@ source $HELPER_SCRIPTS/install.sh # Install latest sbt release -downloadUrl=$(get_github_package_download_url "sbt/sbt" "endswith(\".tgz\")") -download_with_retries "$downloadUrl" "/tmp" "sbt.tgz" -tar zxf /tmp/sbt.tgz -C /usr/share +download_url=$(get_github_package_download_url "sbt/sbt" "endswith(\".tgz\")") +archive_path=$(download_with_retry "$download_url") +tar zxf "$archive_path" -C /usr/share ln -s /usr/share/sbt/bin/sbt /usr/bin/sbt invoke_tests "Tools" "Sbt" diff --git a/images/ubuntu/scripts/build/install-selenium.sh b/images/ubuntu/scripts/build/install-selenium.sh index c24473378..0cbc90408 100644 --- a/images/ubuntu/scripts/build/install-selenium.sh +++ b/images/ubuntu/scripts/build/install-selenium.sh @@ -8,19 +8,17 @@ source $HELPER_SCRIPTS/install.sh source $HELPER_SCRIPTS/etc-environment.sh -# Download Selenium server SELENIUM_MAJOR_VERSION=$(get_toolset_value '.selenium.version') -SELENIUM_BINARY_NAME=$(get_toolset_value '.selenium.binary_name') -SELENIUM_JAR_PATH="/usr/share/java" -SELENIUM_JAR_NAME="$SELENIUM_BINARY_NAME.jar" -SELENIUM_DOWNLOAD_URL=$(get_github_package_download_url "SeleniumHQ/selenium" "contains(\"${SELENIUM_BINARY_NAME}-${SELENIUM_MAJOR_VERSION}\") and endswith(\".jar\")") -download_with_retries $SELENIUM_DOWNLOAD_URL $SELENIUM_JAR_PATH $SELENIUM_JAR_NAME + +# Download Selenium server +SELENIUM_DOWNLOAD_URL=$(get_github_package_download_url "SeleniumHQ/selenium" "contains(\"selenium-server-${SELENIUM_MAJOR_VERSION}\") and endswith(\".jar\")") +SELENIUM_JAR_PATH=$(download_with_retry "$SELENIUM_DOWNLOAD_URL" "/usr/share/java/selenium-server.jar") # Create an epmty file to retrive selenium version -SELENIUM_FULL_VERSION=$(echo $SELENIUM_DOWNLOAD_URL | awk -F"${SELENIUM_BINARY_NAME}-|.jar" '{print $2}') -touch "$SELENIUM_JAR_PATH/$SELENIUM_BINARY_NAME-$SELENIUM_FULL_VERSION" +SELENIUM_FULL_VERSION=$(echo $SELENIUM_DOWNLOAD_URL | awk -F"selenium-server-|.jar" '{print $2}') +touch "/usr/share/java/selenium-server-$SELENIUM_FULL_VERSION" # Add SELENIUM_JAR_PATH environment variable -setEtcEnvironmentVariable "SELENIUM_JAR_PATH" "${SELENIUM_JAR_PATH}/${SELENIUM_JAR_NAME}" +setEtcEnvironmentVariable "SELENIUM_JAR_PATH" "$SELENIUM_JAR_PATH" invoke_tests "Tools" "Selenium" diff --git a/images/ubuntu/scripts/build/install-sqlpackage.sh b/images/ubuntu/scripts/build/install-sqlpackage.sh index 7623bd95c..537d1b29b 100644 --- a/images/ubuntu/scripts/build/install-sqlpackage.sh +++ b/images/ubuntu/scripts/build/install-sqlpackage.sh @@ -10,17 +10,15 @@ source $HELPER_SCRIPTS/os.sh # Install libssl1.1 dependency if isUbuntu22; then - download_with_retries "http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.20_amd64.deb" "/tmp" + libssl_deb_path=$(download_with_retry "http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.20_amd64.deb") libssl_hash="0b3251aee55db6e20d02f4b9a2b703c9874a85ab6a20b12f4870f52f91633d37" - use_checksum_comparison "/tmp/libssl1.1_1.1.1f-1ubuntu2.20_amd64.deb" "${libssl_hash}" - dpkg -i /tmp/libssl1.1_1.1.1f-1ubuntu2.20_amd64.deb + use_checksum_comparison "$libssl_deb_path" "$libssl_hash" + dpkg -i "$libssl_deb_path" fi # Install SqlPackage -download_with_retries "https://aka.ms/sqlpackage-linux" "." "sqlpackage.zip" - -unzip -qq sqlpackage.zip -d /usr/local/sqlpackage -rm -f sqlpackage.zip +archive_path=$(download_with_retry "https://aka.ms/sqlpackage-linux") +unzip -qq "$archive_path" -d /usr/local/sqlpackage chmod +x /usr/local/sqlpackage/sqlpackage ln -sf /usr/local/sqlpackage/sqlpackage /usr/local/bin diff --git a/images/ubuntu/scripts/build/install-swift.sh b/images/ubuntu/scripts/build/install-swift.sh index ca2f6e3f4..64fc6637f 100644 --- a/images/ubuntu/scripts/build/install-swift.sh +++ b/images/ubuntu/scripts/build/install-swift.sh @@ -9,33 +9,33 @@ source $HELPER_SCRIPTS/install.sh source $HELPER_SCRIPTS/etc-environment.sh # Install -image_label="$(lsb_release -rs)" +image_label="ubuntu$(lsb_release -rs)" swift_version=$(curl -fsSL "https://api.github.com/repos/apple/swift/releases/latest" | jq -r '.tag_name | match("[0-9.]+").string') +swift_release_name="swift-${swift_version}-RELEASE-${image_label}" -swift_tar_name="swift-$swift_version-RELEASE-ubuntu$image_label.tar.gz" -swift_tar_url="https://swift.org/builds/swift-$swift_version-release/ubuntu${image_label//./}/swift-$swift_version-RELEASE/$swift_tar_name" -download_with_retries $swift_tar_url "/tmp" "$swift_tar_name" +archive_url="https://swift.org/builds/swift-${swift_version}-release/${image_label//./}/swift-${swift_version}-RELEASE/${swift_release_name}.tar.gz" +archive_path=$(download_with_retry "$archive_url") # Verifing pgp signature using official swift pgp key. Referring to https://www.swift.org/install/linux/#Installation-via-Tarball -# Download swift pgp key -download_with_retries "https://swift.org/keys/all-keys.asc" "/tmp" "all-keys.asc" -# Import swift pgp key -gpg --no-default-keyring --keyring swift --import /tmp/all-keys.asc -# Download signature file -download_with_retries "$swift_tar_url.sig" "/tmp" "$swift_tar_name.sig" -# Verify signature -gpg --no-default-keyring --keyring swift --verify "/tmp/$swift_tar_name.sig" "/tmp/$swift_tar_name" +# Download and import swift pgp key +pgp_key_path=$(download_with_retry "https://swift.org/keys/all-keys.asc") +gpg --no-default-keyring --keyring swift --import "$pgp_key_path" + +# Download and verify signature +signature_path=$(download_with_retry "${archive_url}.sig") +gpg --no-default-keyring --keyring swift --verify "$signature_path" "$archive_path" + # Remove swift pgp public key with temporary keyring rm ~/.gnupg/swift - -tar xzf /tmp/$swift_tar_name +# Extract and install swift +tar xzf "$archive_path" -C /tmp SWIFT_INSTALL_ROOT="/usr/share/swift" SWIFT_BIN_ROOT="$SWIFT_INSTALL_ROOT/usr/bin" SWIFT_LIB_ROOT="$SWIFT_INSTALL_ROOT/usr/lib" -mv swift-$swift_version-RELEASE-ubuntu$image_label $SWIFT_INSTALL_ROOT +mv "/tmp/${swift_release_name}" $SWIFT_INSTALL_ROOT mkdir -p /usr/local/lib ln -s "$SWIFT_BIN_ROOT/swift" /usr/local/bin/swift diff --git a/images/ubuntu/scripts/build/install-terraform.sh b/images/ubuntu/scripts/build/install-terraform.sh index c2da2d631..7392eb839 100644 --- a/images/ubuntu/scripts/build/install-terraform.sh +++ b/images/ubuntu/scripts/build/install-terraform.sh @@ -7,10 +7,8 @@ source $HELPER_SCRIPTS/install.sh # Install Terraform -URL=$(curl -fsSL https://api.releases.hashicorp.com/v1/releases/terraform/latest | jq -r '.builds[] | select((.arch=="amd64") and (.os=="linux")).url') -ZIP_NAME="terraform_linux_amd64.zip" -download_with_retries "${URL}" "/tmp" "${ZIP_NAME}" -unzip -qq "/tmp/${ZIP_NAME}" -d /usr/local/bin -rm -f "/tmp/${ZIP_NAME}" +download_url=$(curl -fsSL https://api.releases.hashicorp.com/v1/releases/terraform/latest | jq -r '.builds[] | select((.arch=="amd64") and (.os=="linux")).url') +archive_path=$(download_with_retry "${download_url}") +unzip -qq "$archive_path" -d /usr/local/bin invoke_tests "Tools" "Terraform" diff --git a/images/ubuntu/scripts/build/install-yq.sh b/images/ubuntu/scripts/build/install-yq.sh index 677a84cb4..f4f26dce8 100644 --- a/images/ubuntu/scripts/build/install-yq.sh +++ b/images/ubuntu/scripts/build/install-yq.sh @@ -1,20 +1,22 @@ #!/bin/bash -e ################################################################################ ## File: install-yq.sh -## Desc: Install YQ -## Supply chain security: YQ - checksum validation +## Desc: Install yq - a command-line YAML, JSON and XML processor +## Supply chain security: yq - checksum validation ################################################################################ # Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh -# Download YQ +# Download yq base_url="https://github.com/mikefarah/yq/releases/latest/download" -download_with_retries "${base_url}/yq_linux_amd64" "/tmp" "yq" -# Supply chain security - YQ +binary_path=$(download_with_retry "${base_url}/yq_linux_amd64") + +# Supply chain security - yq external_hash=$(get_hash_from_remote_file "${base_url}/checksums" "yq_linux_amd64 " "" " " "19") -use_checksum_comparison "/tmp/yq" "${external_hash}" -# Install YQ -sudo install /tmp/yq /usr/bin/yq +use_checksum_comparison "$binary_path" "$external_hash" + +# Install yq +sudo install "$binary_path" /usr/bin/yq invoke_tests "Tools" "yq" diff --git a/images/ubuntu/scripts/build/install-zstd.sh b/images/ubuntu/scripts/build/install-zstd.sh index 9c3b6c593..5f4db375f 100644 --- a/images/ubuntu/scripts/build/install-zstd.sh +++ b/images/ubuntu/scripts/build/install-zstd.sh @@ -10,19 +10,29 @@ source $HELPER_SCRIPTS/install.sh # Download zstd release_tag=$(curl -fsSL https://api.github.com/repos/facebook/zstd/releases/latest | jq -r '.tag_name') -zstd_tar_name=zstd-${release_tag//v}.tar.gz -URL=https://github.com/facebook/zstd/releases/download/${release_tag}/${zstd_tar_name} -download_with_retries "${URL}" "/tmp" "${zstd_tar_name}" +release_name="zstd-${release_tag//v}" +download_url="https://github.com/facebook/zstd/releases/download/${release_tag}/${release_name}.tar.gz" +archive_path=$(download_with_retry "${download_url}") + # Supply chain security - zstd -external_hash=$(get_hash_from_remote_file "${URL}.sha256" "${zstd_tar_name}") -use_checksum_comparison "/tmp/${zstd_tar_name}" "${external_hash}" +external_hash=$(get_hash_from_remote_file "${download_url}.sha256" "${release_name}.tar.gz") +use_checksum_comparison "$archive_path" "$external_hash" + # Install zstd apt-get install -y liblz4-dev -tar xzf /tmp/$zstd_tar_name -C /tmp -make -C /tmp/zstd-${release_tag//v}/contrib/pzstd all -make -C /tmp/zstd-${release_tag//v} zstd-release -for copyprocess in zstd zstdless zstdgrep; do cp /tmp/zstd-${release_tag//v}/programs/$copyprocess /usr/local/bin/; done -cp /tmp/zstd-${release_tag//v}/contrib/pzstd/pzstd /usr/local/bin/ -for symlink in zstdcat zstdmt unzstd; do ln -sf /usr/local/bin/zstd /usr/local/bin/$symlink; done +tar xzf "$archive_path" -C /tmp + +make -C "/tmp/${release_name}/contrib/pzstd" all +make -C "/tmp/${release_name}" zstd-release + +for copyprocess in zstd zstdless zstdgrep; do + cp "/tmp/${release_name}/programs/${copyprocess}" /usr/local/bin/ +done + +cp "/tmp/${release_name}/contrib/pzstd/pzstd" /usr/local/bin/ + +for symlink in zstdcat zstdmt unzstd; do + ln -sf /usr/local/bin/zstd /usr/local/bin/${symlink} +done invoke_tests "Tools" "Zstd" diff --git a/images/ubuntu/scripts/docs-gen/SoftwareReport.Browsers.psm1 b/images/ubuntu/scripts/docs-gen/SoftwareReport.Browsers.psm1 index e3fa61e34..9ae849634 100644 --- a/images/ubuntu/scripts/docs-gen/SoftwareReport.Browsers.psm1 +++ b/images/ubuntu/scripts/docs-gen/SoftwareReport.Browsers.psm1 @@ -34,8 +34,7 @@ function Get-EdgeDriverVersion { } function Get-SeleniumVersion { - $seleniumBinaryName = Get-ToolsetValue "selenium.binary_name" - $fullSeleniumVersion = (Get-ChildItem "/usr/share/java/${seleniumBinaryName}-*").Name -replace "${seleniumBinaryName}-" + $fullSeleniumVersion = (Get-ChildItem "/usr/share/java/selenium-server-*").Name -replace "selenium-server-" return $fullSeleniumVersion } diff --git a/images/ubuntu/scripts/helpers/install.sh b/images/ubuntu/scripts/helpers/install.sh index ee918f2b6..0f803f04b 100644 --- a/images/ubuntu/scripts/helpers/install.sh +++ b/images/ubuntu/scripts/helpers/install.sh @@ -4,50 +4,45 @@ ## Desc: Helper functions for installing tools ################################################################################ -download_with_retries() { -# Due to restrictions of bash functions, positional arguments are used here. -# In case if you using latest argument NAME, you should also set value to all previous parameters. -# Example: download_with_retries $ANDROID_SDK_URL "." "android_sdk.zip" - local URL="$1" - local DEST="${2:-.}" - local NAME="${3:-${URL##*/}}" - local COMPRESSED="$4" +download_with_retry() { + url=$1 + download_path=$2 - if [[ $COMPRESSED == "compressed" ]]; then - local COMMAND="curl $URL -4 -sL --compressed -o '$DEST/$NAME' -w '%{http_code}'" - else - local COMMAND="curl $URL -4 -sL -o '$DEST/$NAME' -w '%{http_code}'" + if [ -z "$download_path" ]; then + download_path="/tmp/$(basename "$url")" fi - # Save current errexit state and disable it to prevent unexpected exit on error - if echo $SHELLOPTS | grep '\(^\|:\)errexit\(:\|$\)' > /dev/null; - then - local ERR_EXIT_ENABLED=true - else - local ERR_EXIT_ENABLED=false - fi - set +e + echo "Downloading package from $url to $download_path..." >&2 - echo "Downloading '$URL' to '${DEST}/${NAME}'..." - retries=20 interval=30 - while [ $retries -gt 0 ]; do - ((retries--)) - test "$ERR_EXIT_ENABLED" = true && set +e - http_code=$(eval $COMMAND) - exit_code=$? - test "$ERR_EXIT_ENABLED" = true && set -e - if [ $http_code -eq 200 ] && [ $exit_code -eq 0 ]; then - echo "Download completed" - return 0 + download_start_time=$(date +%s) + + for ((retries=20; retries>0; retries--)); do + attempt_start_time=$(date +%s) + if http_code=$(curl -4sSLo "$download_path" "$url" -w '%{http_code}'); then + attempt_seconds=$(($(date +%s) - attempt_start_time)) + if [ "$http_code" -eq 200 ]; then + echo "Package downloaded in $attempt_seconds seconds" >&2 + break + else + echo "Received HTTP status code $http_code after $attempt_seconds seconds" >&2 + fi else - echo "Error — Either HTTP response code for '$URL' is wrong - '$http_code' or exit code is not 0 - '$exit_code'. Waiting $interval seconds before the next attempt, $retries attempts left" - sleep $interval + attempt_seconds=$(($(date +%s) - attempt_start_time)) + echo "Package download failed in $attempt_seconds seconds" >&2 fi + + if [ "$retries" -le 1 ]; then + total_seconds=$(($(date +%s) - download_start_time)) + echo "Package download failed after $total_seconds seconds" >&2 + exit 1 + fi + + echo "Waiting $interval seconds before retrying (retries left: $retries)..." >&2 + sleep $interval done - echo "Could not download $URL" - return 1 + echo "$download_path" } ## Use dpkg to figure out if a package has already been installed diff --git a/images/ubuntu/scripts/tests/Tools.Tests.ps1 b/images/ubuntu/scripts/tests/Tools.Tests.ps1 index e420314ef..34662055f 100644 --- a/images/ubuntu/scripts/tests/Tools.Tests.ps1 +++ b/images/ubuntu/scripts/tests/Tools.Tests.ps1 @@ -208,8 +208,7 @@ Describe "Sbt" { Describe "Selenium" { It "Selenium is installed" { - $seleniumBinaryName = (Get-ToolsetContent).selenium.binary_name - $seleniumPath = Join-Path "/usr/share/java" "$seleniumBinaryName.jar" + $seleniumPath = Join-Path "/usr/share/java" "selenium-server.jar" $seleniumPath | Should -Exist } } diff --git a/images/ubuntu/toolsets/toolset-2004.json b/images/ubuntu/toolsets/toolset-2004.json index 6200b701e..9b026e410 100644 --- a/images/ubuntu/toolsets/toolset-2004.json +++ b/images/ubuntu/toolsets/toolset-2004.json @@ -298,8 +298,7 @@ {"name": "fastlane"} ], "selenium": { - "version": "4", - "binary_name": "selenium-server" + "version": "4" }, "node": { "default": "18" diff --git a/images/ubuntu/toolsets/toolset-2204.json b/images/ubuntu/toolsets/toolset-2204.json index d6c699437..b4213e10e 100644 --- a/images/ubuntu/toolsets/toolset-2204.json +++ b/images/ubuntu/toolsets/toolset-2204.json @@ -289,8 +289,7 @@ {"name": "fastlane"} ], "selenium": { - "version": "4", - "binary_name": "selenium-server" + "version": "4" }, "node": { "default": "18"