From 8437ff39903ee7b348688a24d7a2606883347a59 Mon Sep 17 00:00:00 2001 From: Shamil Mubarakshin <127750046+shamil-mubarakshin@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:01:37 +0100 Subject: [PATCH] [ubuntu] Refactor GitHub package download URL function (#8947) * [ubuntu] Refactor Github Package URL function * Fix function and selenium version * Remove beta/release candidate versions --- .../scripts/build/install-actions-cache.sh | 2 +- .../scripts/build/install-aliyun-cli.sh | 2 +- images/ubuntu/scripts/build/install-cmake.sh | 4 +- images/ubuntu/scripts/build/install-docker.sh | 4 +- .../ubuntu/scripts/build/install-firefox.sh | 2 +- .../scripts/build/install-github-cli.sh | 4 +- images/ubuntu/scripts/build/install-kotlin.sh | 2 +- .../scripts/build/install-kubernetes-tools.sh | 2 +- .../ubuntu/scripts/build/install-oras-cli.sh | 4 +- .../scripts/build/install-runner-package.sh | 2 +- images/ubuntu/scripts/build/install-sbt.sh | 2 +- .../ubuntu/scripts/build/install-selenium.sh | 2 +- images/ubuntu/scripts/helpers/install.sh | 45 +++++++++++++------ 13 files changed, 48 insertions(+), 29 deletions(-) diff --git a/images/ubuntu/scripts/build/install-actions-cache.sh b/images/ubuntu/scripts/build/install-actions-cache.sh index 416dab22a..1761c8a0b 100644 --- a/images/ubuntu/scripts/build/install-actions-cache.sh +++ b/images/ubuntu/scripts/build/install-actions-cache.sh @@ -17,7 +17,7 @@ echo "Setting up ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE variable to ${ACTION_ARCHIV setEtcEnvironmentVariable "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE" "${ACTION_ARCHIVE_CACHE_DIR}" # Download latest release from github.com/actions/action-versions and untar to /opt/actionarchivecache -downloadUrl=$(get_github_package_download_url "actions/action-versions" "contains(\"action-versions.tar.gz\")") +downloadUrl=$(resolve_github_release_asset_url "actions/action-versions" "contains(\"action-versions.tar.gz\")" "latest") archive_path=$(download_with_retry "$downloadUrl") tar -xzf "$archive_path" -C $ACTION_ARCHIVE_CACHE_DIR diff --git a/images/ubuntu/scripts/build/install-aliyun-cli.sh b/images/ubuntu/scripts/build/install-aliyun-cli.sh index d3d8813bf..2603ae3d5 100644 --- a/images/ubuntu/scripts/build/install-aliyun-cli.sh +++ b/images/ubuntu/scripts/build/install-aliyun-cli.sh @@ -15,7 +15,7 @@ if isUbuntu20; then toolset_version=$(get_toolset_value '.aliyunCli.version') download_url="https://github.com/aliyun/aliyun-cli/releases/download/v$toolset_version/aliyun-cli-linux-$toolset_version-amd64.tgz" else - download_url=$(get_github_package_download_url "aliyun/aliyun-cli" "contains(\"aliyun-cli-linux\") and endswith(\"amd64.tgz\")") + download_url=$(resolve_github_release_asset_url "aliyun/aliyun-cli" "contains(\"aliyun-cli-linux\") and endswith(\"amd64.tgz\")" "latest") hash_url="https://github.com/aliyun/aliyun-cli/releases/latest/download/SHASUMS256.txt" fi diff --git a/images/ubuntu/scripts/build/install-cmake.sh b/images/ubuntu/scripts/build/install-cmake.sh index fed29cba8..373388bca 100644 --- a/images/ubuntu/scripts/build/install-cmake.sh +++ b/images/ubuntu/scripts/build/install-cmake.sh @@ -14,10 +14,10 @@ if command -v cmake; then echo "cmake is already installed" else # Download script to install CMake - download_url=$(get_github_package_download_url "Kitware/CMake" "endswith(\"inux-x86_64.sh\")") + download_url=$(resolve_github_release_asset_url "Kitware/CMake" "endswith(\"inux-x86_64.sh\")" "latest") curl -fsSL "${download_url}" -o cmakeinstall.sh # Supply chain security - CMake - hash_url=$(get_github_package_download_url "Kitware/CMake" "endswith(\"SHA-256.txt\")") + hash_url=$(resolve_github_release_asset_url "Kitware/CMake" "endswith(\"SHA-256.txt\")" "latest") external_hash=$(get_hash_from_remote_file "$hash_url" "linux-x86_64.sh") use_checksum_comparison "cmakeinstall.sh" "$external_hash" # Install CMake and remove the install script diff --git a/images/ubuntu/scripts/build/install-docker.sh b/images/ubuntu/scripts/build/install-docker.sh index bbf77048f..967dd3567 100644 --- a/images/ubuntu/scripts/build/install-docker.sh +++ b/images/ubuntu/scripts/build/install-docker.sh @@ -19,10 +19,10 @@ apt-get update apt-get install --no-install-recommends docker-ce docker-ce-cli containerd.io docker-buildx-plugin # Download docker compose v2 from releases -URL=$(get_github_package_download_url "docker/compose" "contains(\"compose-linux-x86_64\")") +URL=$(resolve_github_release_asset_url "docker/compose" "contains(\"compose-linux-x86_64\")" "latest") curl -fsSL "${URL}" -o /tmp/docker-compose # Supply chain security - Docker Compose v2 -compose_hash_url=$(get_github_package_download_url "docker/compose" "contains(\"checksums.txt\")") +compose_hash_url=$(resolve_github_release_asset_url "docker/compose" "contains(\"checksums.txt\")" "latest") compose_external_hash=$(get_hash_from_remote_file "${compose_hash_url}" "compose-linux-x86_64") use_checksum_comparison "/tmp/docker-compose" "${compose_external_hash}" # Install docker compose v2 diff --git a/images/ubuntu/scripts/build/install-firefox.sh b/images/ubuntu/scripts/build/install-firefox.sh index 53714ca1c..337d74339 100644 --- a/images/ubuntu/scripts/build/install-firefox.sh +++ b/images/ubuntu/scripts/build/install-firefox.sh @@ -33,7 +33,7 @@ echo "mozillateam $repo_url" >> $HELPER_SCRIPTS/apt-sources.txt echo 'pref("intl.locale.requested","en_US");' >> "/usr/lib/firefox/browser/defaults/preferences/syspref.js" # Download and unpack latest release of geckodriver -download_url=$(get_github_package_download_url "mozilla/geckodriver" "test(\"linux64.tar.gz$\")") +download_url=$(resolve_github_release_asset_url "mozilla/geckodriver" "test(\"linux64.tar.gz$\")" "latest") driver_archive_path=$(download_with_retry "$download_url") GECKODRIVER_DIR="/usr/local/share/gecko_driver" diff --git a/images/ubuntu/scripts/build/install-github-cli.sh b/images/ubuntu/scripts/build/install-github-cli.sh index 6e43aed4a..dc14febaa 100644 --- a/images/ubuntu/scripts/build/install-github-cli.sh +++ b/images/ubuntu/scripts/build/install-github-cli.sh @@ -10,10 +10,10 @@ source $HELPER_SCRIPTS/install.sh # Download GitHub CLI -gh_cli_url=$(get_github_package_download_url "cli/cli" "contains(\"linux\") and contains(\"amd64\") and contains(\".deb\")") +gh_cli_url=$(resolve_github_release_asset_url "cli/cli" "contains(\"linux\") and contains(\"amd64\") and contains(\".deb\")" "latest") gh_cli_deb_path=$(download_with_retry "$gh_cli_url") # Supply chain security - GitHub CLI -hash_url=$(get_github_package_download_url "cli/cli" "contains(\"checksums.txt\")") +hash_url=$(resolve_github_release_asset_url "cli/cli" "contains(\"checksums.txt\")" "latest") external_hash=$(get_hash_from_remote_file "$hash_url" "linux_amd64.deb") use_checksum_comparison "$gh_cli_deb_path" "$external_hash" # Install GitHub CLI diff --git a/images/ubuntu/scripts/build/install-kotlin.sh b/images/ubuntu/scripts/build/install-kotlin.sh index a8c2bd273..3b144443a 100644 --- a/images/ubuntu/scripts/build/install-kotlin.sh +++ b/images/ubuntu/scripts/build/install-kotlin.sh @@ -8,7 +8,7 @@ source $HELPER_SCRIPTS/install.sh KOTLIN_ROOT="/usr/share" -download_url=$(get_github_package_download_url "JetBrains/kotlin" "contains(\"kotlin-compiler\")") +download_url=$(resolve_github_release_asset_url "JetBrains/kotlin" "contains(\"kotlin-compiler\")" "latest") archive_path=$(download_with_retry "$download_url") # Supply chain security - Kotlin diff --git a/images/ubuntu/scripts/build/install-kubernetes-tools.sh b/images/ubuntu/scripts/build/install-kubernetes-tools.sh index 0806c9814..03383065c 100644 --- a/images/ubuntu/scripts/build/install-kubernetes-tools.sh +++ b/images/ubuntu/scripts/build/install-kubernetes-tools.sh @@ -9,7 +9,7 @@ source $HELPER_SCRIPTS/install.sh # Download KIND -kind_url=$(get_github_package_download_url "kubernetes-sigs/kind" "contains(\"kind-linux-amd64\")") +kind_url=$(resolve_github_release_asset_url "kubernetes-sigs/kind" "contains(\"kind-linux-amd64\")" "latest") curl -fsSL -o /tmp/kind "${kind_url}" # Supply chain security - KIND kind_external_hash=$(get_hash_from_remote_file "${kind_url}.sha256sum" "kind-linux-amd64") diff --git a/images/ubuntu/scripts/build/install-oras-cli.sh b/images/ubuntu/scripts/build/install-oras-cli.sh index 352a94b73..43cfefc93 100644 --- a/images/ubuntu/scripts/build/install-oras-cli.sh +++ b/images/ubuntu/scripts/build/install-oras-cli.sh @@ -8,13 +8,13 @@ source $HELPER_SCRIPTS/install.sh # Determine latest ORAS CLI version -download_url=$(get_github_package_download_url "oras-project/oras" "endswith(\"linux_amd64.tar.gz\")") +download_url=$(resolve_github_release_asset_url "oras-project/oras" "endswith(\"linux_amd64.tar.gz\")" "latest") # Download ORAS CLI archive_path=$(download_with_retry "$download_url") # Supply chain security - ORAS CLI -hash_url=$(get_github_package_download_url "oras-project/oras" "contains(\"checksums.txt\")") +hash_url=$(resolve_github_release_asset_url "oras-project/oras" "contains(\"checksums.txt\")" "latest") external_hash=$(get_hash_from_remote_file "${hash_url}" "linux_amd64.tar.gz") use_checksum_comparison "$archive_path" "${external_hash}" diff --git a/images/ubuntu/scripts/build/install-runner-package.sh b/images/ubuntu/scripts/build/install-runner-package.sh index 967c0466e..b18fdbc49 100644 --- a/images/ubuntu/scripts/build/install-runner-package.sh +++ b/images/ubuntu/scripts/build/install-runner-package.sh @@ -7,7 +7,7 @@ # Source the helpers for use with the script source $HELPER_SCRIPTS/install.sh -download_url=$(get_github_package_download_url "actions/runner" 'test("actions-runner-linux-x64-[0-9]+\\.[0-9]{3}\\.[0-9]+\\.tar\\.gz")') +download_url=$(resolve_github_release_asset_url "actions/runner" 'test("actions-runner-linux-x64-[0-9]+\\.[0-9]{3}\\.[0-9]+\\.tar\\.gz")' "latest") archive_name="${download_url##*/}" archive_path=$(download_with_retry "$download_url") diff --git a/images/ubuntu/scripts/build/install-sbt.sh b/images/ubuntu/scripts/build/install-sbt.sh index ae7610f2d..037b8e512 100644 --- a/images/ubuntu/scripts/build/install-sbt.sh +++ b/images/ubuntu/scripts/build/install-sbt.sh @@ -7,7 +7,7 @@ source $HELPER_SCRIPTS/install.sh # Install latest sbt release -download_url=$(get_github_package_download_url "sbt/sbt" "endswith(\".tgz\")") +download_url=$(resolve_github_release_asset_url "sbt/sbt" "endswith(\".tgz\")" "latest") archive_path=$(download_with_retry "$download_url") tar zxf "$archive_path" -C /usr/share ln -s /usr/share/sbt/bin/sbt /usr/bin/sbt diff --git a/images/ubuntu/scripts/build/install-selenium.sh b/images/ubuntu/scripts/build/install-selenium.sh index 0cbc90408..fa30d5e7d 100644 --- a/images/ubuntu/scripts/build/install-selenium.sh +++ b/images/ubuntu/scripts/build/install-selenium.sh @@ -11,7 +11,7 @@ source $HELPER_SCRIPTS/etc-environment.sh SELENIUM_MAJOR_VERSION=$(get_toolset_value '.selenium.version') # Download Selenium server -SELENIUM_DOWNLOAD_URL=$(get_github_package_download_url "SeleniumHQ/selenium" "contains(\"selenium-server-${SELENIUM_MAJOR_VERSION}\") and endswith(\".jar\")") +SELENIUM_DOWNLOAD_URL=$(resolve_github_release_asset_url "SeleniumHQ/selenium" "contains(\"selenium-server-\") and endswith(\".jar\")" "$SELENIUM_MAJOR_VERSION\.*") SELENIUM_JAR_PATH=$(download_with_retry "$SELENIUM_DOWNLOAD_URL" "/usr/share/java/selenium-server.jar") # Create an epmty file to retrive selenium version diff --git a/images/ubuntu/scripts/helpers/install.sh b/images/ubuntu/scripts/helpers/install.sh index 52288b93b..2add22aa0 100644 --- a/images/ubuntu/scripts/helpers/install.sh +++ b/images/ubuntu/scripts/helpers/install.sh @@ -66,26 +66,45 @@ get_toolset_value() { echo "$(jq -r "$query" $toolset_path)" } -get_github_package_download_url() { - local REPO_ORG=$1 - local FILTER=$2 - local VERSION=$3 - local SEARCH_IN_COUNT="100" +resolve_github_release_asset_url() { + local repo=$1 + local filter=$2 + local version=${3:-"*"} + local allow_pre_release=${4:-false} - json=$(curl -fsSL "https://api.github.com/repos/${REPO_ORG}/releases?per_page=${SEARCH_IN_COUNT}") + page_size="100" - if [ -n "$VERSION" ]; then - tagName=$(echo $json | jq -r '.[] | select(.prerelease==false).tag_name' | sort --unique --version-sort | egrep -v ".*-[a-z]|beta" | egrep "\w*${VERSION}" | tail -1) + json=$(curl -fsSL "https://api.github.com/repos/${repo}/releases?per_page=${page_size}") + + if [[ $allow_pre_release == "true" ]]; then + json=$(echo $json | jq -r '.[] | select(.assets | length > 0)') else - tagName=$(echo $json | jq -r '.[] | select((.prerelease==false) and (.assets | length > 0)).tag_name' | sort --unique --version-sort | egrep -v ".*-[a-z]|beta" | tail -1) + json=$(echo $json | jq -r '.[] | select((.prerelease==false) and (.assets | length > 0))') fi - downloadUrl=$(echo $json | jq -r ".[] | select(.tag_name==\"${tagName}\").assets[].browser_download_url | select(${FILTER})" | head -n 1) - if [ -z "$downloadUrl" ]; then - echo "Failed to parse a download url for the '${tagName}' tag using '${FILTER}' filter" + if [[ $version == "latest" ]]; then + tag_name=$(echo $json | jq -r '.tag_name' | sort --unique --version-sort | egrep -v ".*-[a-z]|beta" | tail -n 1) + elif [[ $version == *"*"* ]]; then + tag_name=$(echo $json | jq -r '.tag_name' | sort --unique --version-sort | egrep -v ".*-[a-z]|beta" | egrep "${version}" | tail -n 1) + else + tag_names=$(echo $json | jq -r '.tag_name' | sort --unique --version-sort | egrep -v ".*-[a-z]|beta" | egrep "${version}") + + for element in $tag_names; do + semver=$(echo "$element" | awk 'match($0, /[0-9]+\.[0-9]+\.[0-9]+/) {print substr($0, RSTART, RLENGTH)}') + + if [[ $semver == $version ]]; then + tag_name=$element + fi + done + fi + + download_url=$(echo $json | jq -r ". | select(.tag_name==\"${tag_name}\").assets[].browser_download_url | select(${filter})" | head -n 1) + if [ -z "$download_url" ]; then + echo "Failed to parse a download url for the '${tag_name}' tag using '${filter}' filter" exit 1 fi - echo $downloadUrl + + echo $download_url } get_github_package_hash() {