[Ubuntu] Pre-cache Ruby binaries (#2084)

* precache ruby binaries

* remove github_feed_token

* fix logs

images/linux/scripts/installers/Validate-Toolset.ps1

@@ -43,6 +43,10 @@ $toolsExecutables = @{
         tools = @("bin/go")
         command = "version"
     }
+    Ruby = @{
+        tools = @("bin/ruby")
+        command = "--version"
+    }
 }
 
 # Get toolset content

images/linux/scripts/installers/hosted-tool-cache.sh

@@ -1,39 +0,0 @@
-#!/bin/bash -e
-################################################################################
-##  File:  hosted-tool-cache.sh
-##  Desc:  Downloads and installs hosted tools cache
-################################################################################
-
-TOOLCACHE_REGISTRY="npm.pkg.github.com"
-
-echo "Configure npm to use github package registry for '@actions' scope"
-npm config set @actions:registry "https://${TOOLCACHE_REGISTRY}"
-
-# Execute in /imagegeneration/installers folder to avoid node_modules creation in $HOME
-pushd $INSTALLER_SCRIPT_FOLDER
-
-# GitHub Package Registry doesn't support downloading public packages without auth so we have to authorize
-echo "Configure auth for github package registry"
-echo "//${TOOLCACHE_REGISTRY}/:_authToken=${GITHUB_FEED_TOKEN}" > ".npmrc"
-
-echo "Installing npm-toolcache..."
-TOOLSET_PATH="$INSTALLER_SCRIPT_FOLDER/toolcache.json"
-
-PACKAGE_LIST=($(jq -r 'keys | .[]' $TOOLSET_PATH))
-
-for PACKAGE_NAME in ${PACKAGE_LIST[@]}; do
-    PACKAGE_VERSIONS=($(jq -r ".[\"$PACKAGE_NAME\"] | .[]" $TOOLSET_PATH))
-    for PACKAGE_VERSION in ${PACKAGE_VERSIONS[@]}; do
-        echo "Install ${PACKAGE_NAME}@${PACKAGE_VERSION}"
-        npm install ${PACKAGE_NAME}@${PACKAGE_VERSION}
-
-        exit_code=$?
-        if [ $exit_code -ne 0 ]; then
-            echo "${PACKAGE_NAME}@${PACKAGE_VERSION} installation failure;  Error:${exit_code}"
-
-            exit 1
-        fi
-    done;
-done;
-
-popd

images/linux/scripts/installers/ruby.sh

@@ -4,8 +4,44 @@
 ##  Desc:  Installs Ruby requirements
 ################################################################################
 
+source $HELPER_SCRIPTS/install.sh
+
 sudo apt-get install ruby-full
 sudo gem update --system
 
 # Install Ruby requirements
 apt-get install -y libz-dev openssl libssl-dev
+
+echo "Install Ruby from toolset..."
+toolset="$INSTALLER_SCRIPT_FOLDER/toolset.json"
+PACKAGE_TAR_NAMES=$(curl -s "https://api.github.com/repos/ruby/ruby-builder/releases/latest" | jq -r '.assets[].name')
+TOOLSET_VERSIONS=$(jq -r '.toolcache[] | select(.name | contains("Ruby")) | .versions[]' $toolset)
+PLATFORM_VERSION=$(jq -r '.toolcache[] | select(.name | contains("Ruby")) | .platform_version' $toolset)
+RUBY_PATH="$AGENT_TOOLSDIRECTORY/Ruby"
+
+echo "Check if Ruby hostedtoolcache folder exist..."
+if [ ! -d $RUBY_PATH ]; then
+    mkdir -p $RUBY_PATH
+fi
+
+for TOOLSET_VERSION in ${TOOLSET_VERSIONS[@]}; do
+    PACKAGE_TAR_NAME=$(echo "$PACKAGE_TAR_NAMES" | grep "^ruby-${TOOLSET_VERSION}-ubuntu-${PLATFORM_VERSION}.tar.gz$" | sort -V | tail -1)
+    RUBY_VERSION=$(echo "$PACKAGE_TAR_NAME" | cut -d'-' -f 2)
+    RUBY_VERSION_PATH="$RUBY_PATH/$RUBY_VERSION"
+
+    echo "Create Ruby $RUBY_VERSION directory..."
+    mkdir -p $RUBY_VERSION_PATH
+
+    echo "Downloading tar archive $PACKAGE_TAR_NAME"
+    DOWNLOAD_URL="https://github.com/ruby/ruby-builder/releases/download/toolcache/${PACKAGE_TAR_NAME}"
+    download_with_retries $DOWNLOAD_URL "/tmp" $PACKAGE_TAR_NAME
+
+    echo "Expand '$PACKAGE_TAR_NAME' to the '$RUBY_VERSION_PATH' folder"
+    tar xf "/tmp/$PACKAGE_TAR_NAME" -C $RUBY_VERSION_PATH
+
+    COMPLETE_FILE_PATH="$RUBY_VERSION_PATH/x64.complete"
+    if [ ! -f $COMPLETE_FILE_PATH ]; then
+        echo "Create complete file"
+        touch $COMPLETE_FILE_PATH
+    fi
+done

images/linux/scripts/installers/test-toolcache.sh

@@ -1,69 +0,0 @@
-#!/bin/bash -e
-################################################################################
-##  File:  test-toolcache.sh
-##  Desc:  Test Python and Ruby versions in tools cache
-################################################################################
-
-# Must be procecessed after tool cache setup(hosted-tool-cache.sh).
-
-Test_Hostedtoolcache_Tool() {
-   TOOL_NAME=$1
-   TOOL_EXEC_TEST=$2
-
-   if [ -d "$AGENT_TOOLSDIRECTORY/$TOOL_NAME" ]; then
-      cd $AGENT_TOOLSDIRECTORY/$TOOL_NAME
-
-      tool_dirs=($(find . -mindepth 1 -maxdepth 1 -type d | sed "s|^\./||"))
-
-      echo "--------------------------------------------"
-      echo "$TOOL_NAME versions folders: ${tool_dirs[@]}"
-      echo "--------------------------------------------"
-
-      if [ -n "$tool_dirs" ]; then
-         tool_key=$(echo $TOOL_NAME | tr "[:upper:]" "[:lower:]")
-         package_versions=($(jq -r ".[\"${TOOLCACHE_KEY_VALUE[$tool_key]}\"] | .[]" "$INSTALLER_SCRIPT_FOLDER/toolcache.json"))
-
-         for tool_version in ${package_versions[@]}
-         do
-            version_dir=$(find . -name "$tool_version.*" -print -quit)
-
-            echo "Test $AGENT_TOOLSDIRECTORY/$TOOL_NAME/$version_dir:"
-
-            actual_version=$(eval $AGENT_TOOLSDIRECTORY/$TOOL_NAME/$version_dir/$TOOL_EXEC_TEST)
-
-            if [ "$actual_version" = "$tool_version" ]; then
-                  echo "Passed!"
-            else
-                  echo "Expected: $tool_version; Actual: $actual_version"
-
-                  exit 1
-            fi
-         done;
-      else
-         echo "$AGENT_TOOLSDIRECTORY/$tool_version does not include any folders"
-
-         exit 1
-      fi
-   else
-      echo "$AGENT_TOOLSDIRECTORY/$tool_version does not exist"
-
-      exit 1
-   fi
-}
-
-# define dictionary for key_alias and its values
-declare -A TOOLCACHE_KEY_VALUE
-
-package_list=($(jq -r 'keys | .[]' "$INSTALLER_SCRIPT_FOLDER/toolcache.json"))
-
-for package_name in ${package_list[@]}; do
-   # get key alias
-   key_alias=$(echo $package_name | cut -f2 -d-)
-
-   # set dictionary
-   TOOLCACHE_KEY_VALUE+=(["$key_alias"]="$package_name")
-done;
-
-AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache
-
-Test_Hostedtoolcache_Tool "Ruby" "x64/bin/ruby -e 'puts RUBY_VERSION' | egrep -o '[0-9]+\.[0-9]+'"

images/linux/toolsets/toolcache-1604.json

@@ -1,5 +0,0 @@
-{
-    "@actions/toolcache-ruby-ubuntu-1604-x64": [
-        "2.4", "2.5", "2.6", "2.7"
-    ]
-}

images/linux/toolsets/toolcache-1804.json

@@ -1,5 +0,0 @@
-{
-    "@actions/toolcache-ruby-ubuntu-1804-x64": [
-        "2.4", "2.5", "2.6", "2.7"
-    ]
-}

images/linux/toolsets/toolcache-2004.json

@@ -1,5 +0,0 @@
-{
-    "@actions/toolcache-ruby-ubuntu-2004-x64": [
-        "2.5", "2.6", "2.7"
-    ]
-}

images/linux/toolsets/toolset-1604.json

@@ -57,6 +57,17 @@
             "versions": [
                 "1.72.0"
             ]
+        },
+        {
+            "name": "Ruby",
+            "platform_version": "16.04",
+            "arch": "x64",
+            "versions": [
+                "2.4.*",
+                "2.5.*",
+                "2.6.*",
+                "2.7.*"
+            ]
         }
     ],
     "android": {

images/linux/toolsets/toolset-1804.json

@@ -57,6 +57,17 @@
             "versions": [
                 "1.72.0"
             ]
+        },
+        {
+            "name": "Ruby",
+            "platform_version": "18.04",
+            "arch": "x64",
+            "versions": [
+                "2.4.*",
+                "2.5.*",
+                "2.6.*",
+                "2.7.*"
+            ]
         }
     ],
     "android": {

images/linux/toolsets/toolset-2004.json

@@ -46,6 +46,16 @@
                 "1.15.*"
             ],
             "default": "1.14.*"
+        },
+        {
+            "name": "Ruby",
+            "platform_version": "20.04",
+            "arch": "x64",
+            "versions": [
+                "2.5.*",
+                "2.6.*",
+                "2.7.*"
+            ]
         }
     ],
     "android": {

images/linux/ubuntu1604.json

@@ -20,12 +20,10 @@
         "capture_name_prefix": "packer",
         "image_version": "dev",
         "image_os": "ubuntu16",
-        "github_feed_token": null,
         "run_validation_diskspace": "false"
     },
     "sensitive-variables": [
-        "client_secret",
+        "client_secret"
-        "github_feed_token"
     ],
     "builders": [
         {
@@ -106,11 +104,6 @@
             "source": "{{ template_dir }}/scripts/SoftwareReport",
             "destination": "{{user `image_folder`}}"
         },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolcache-1604.json",
-            "destination": "{{user `installer_script_folder`}}/toolcache.json"
-        },
         {
             "type": "file",
             "source": "{{template_dir}}/toolsets/toolset-1604.json",
@@ -218,15 +211,12 @@
                 "{{template_dir}}/scripts/installers/netlify.sh",
                 "{{template_dir}}/scripts/installers/android.sh",
                 "{{template_dir}}/scripts/installers/azpowershell.sh",
-                "{{template_dir}}/scripts/installers/hosted-tool-cache.sh",
                 "{{template_dir}}/scripts/installers/pypy.sh",
-                "{{template_dir}}/scripts/installers/python.sh",
+                "{{template_dir}}/scripts/installers/python.sh"
-                "{{template_dir}}/scripts/installers/test-toolcache.sh"
             ],
             "environment_vars": [
                 "HELPER_SCRIPTS={{user `helper_script_folder`}}",
                 "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
-                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}",
                 "DEBIAN_FRONTEND=noninteractive"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"

images/linux/ubuntu1804.json

@@ -20,12 +20,10 @@
         "capture_name_prefix": "packer",
         "image_version": "dev",
         "image_os": "ubuntu18",
-        "github_feed_token": null,
         "run_validation_diskspace": "false"
     },
     "sensitive-variables": [
-        "client_secret",
+        "client_secret"
-        "github_feed_token"
     ],
     "builders": [
         {
@@ -109,11 +107,6 @@
             "source": "{{ template_dir }}/scripts/SoftwareReport",
             "destination": "{{user `image_folder`}}"
         },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolcache-1804.json",
-            "destination": "{{user `installer_script_folder`}}/toolcache.json"
-        },
         {
             "type": "file",
             "source": "{{template_dir}}/toolsets/toolset-1804.json",
@@ -222,15 +215,12 @@
                 "{{template_dir}}/scripts/installers/netlify.sh",
                 "{{template_dir}}/scripts/installers/android.sh",
                 "{{template_dir}}/scripts/installers/azpowershell.sh",
-                "{{template_dir}}/scripts/installers/hosted-tool-cache.sh",
                 "{{template_dir}}/scripts/installers/pypy.sh",
-                "{{template_dir}}/scripts/installers/python.sh",
+                "{{template_dir}}/scripts/installers/python.sh"
-                "{{template_dir}}/scripts/installers/test-toolcache.sh"
             ],
             "environment_vars": [
                 "HELPER_SCRIPTS={{user `helper_script_folder`}}",
                 "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
-                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}",
                 "DEBIAN_FRONTEND=noninteractive"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"

images/linux/ubuntu2004.json

@@ -20,14 +20,12 @@
         "capture_name_prefix": "packer",
         "image_version": "dev",
         "image_os": "ubuntu20",
-        "github_feed_token": null,
         "run_validation_diskspace": "false",
         "go_default": "1.14",
         "go_versions": "1.14"
     },
     "sensitive-variables": [
-        "client_secret",
+        "client_secret"
-        "github_feed_token"
     ],
     "builders": [
         {
@@ -119,11 +117,6 @@
             "source": "{{ template_dir }}/scripts/SoftwareReport",
             "destination": "{{user `image_folder`}}"
         },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolcache-2004.json",
-            "destination": "{{user `installer_script_folder`}}/toolcache.json"
-        },
         {
             "type": "file",
             "source": "{{template_dir}}/toolsets/toolset-2004.json",
@@ -232,15 +225,12 @@
                 "{{template_dir}}/scripts/installers/netlify.sh",
                 "{{template_dir}}/scripts/installers/android.sh",
                 "{{template_dir}}/scripts/installers/azpowershell.sh",
-                "{{template_dir}}/scripts/installers/hosted-tool-cache.sh",
                 "{{template_dir}}/scripts/installers/pypy.sh",
-                "{{template_dir}}/scripts/installers/python.sh",
+                "{{template_dir}}/scripts/installers/python.sh"
-                "{{template_dir}}/scripts/installers/test-toolcache.sh"
             ],
             "environment_vars": [
                 "HELPER_SCRIPTS={{user `helper_script_folder`}}",
                 "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
-                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}",
                 "DEBIAN_FRONTEND=noninteractive"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"