Toolcache: Add NPM Toolcache support for Windows/Linux

images/linux/scripts/installers/1804/hosted-tool-cache.sh

@@ -1,54 +0,0 @@
-#!/bin/bash
-################################################################################
-##  File:  hosted-tool-cache.sh
-##  Desc:  Downloads and installs hosted tools cache
-################################################################################
-
-# Source the helpers for use with the script
-source $HELPER_SCRIPTS/document.sh
-
-# Fail out if any setups fail
-set -e
-
-# Download hosted tool cache
-AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache
-echo "AGENT_TOOLSDIRECTORY=$AGENT_TOOLSDIRECTORY" | tee -a /etc/environment
-azcopy --recursive \
-       --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/ubuntu-1804 \
-       --destination $AGENT_TOOLSDIRECTORY
-
-# Install tools from hosted tool cache
-original_directory=$PWD
-setups=$(find $AGENT_TOOLSDIRECTORY -name setup.sh)
-for setup in $setups; do
-	chmod +x $setup;
-	cd $(dirname $setup);
-	./$(basename $setup);
-	cd $original_directory;
-done;
-
-DocumentInstalledItem "Python:"
-pythons=$(ls $AGENT_TOOLSDIRECTORY/Python)
-for python in $pythons; do
-	DocumentInstalledItemIndent "Python $python"
-done;
-
-# PyPy is also configured using the setup-python action
-pypys=$(ls $AGENT_TOOLSDIRECTORY/PyPy)
-for pypy in $pypys; do
-	DocumentInstalledItemIndent "PyPy $pypy"
-	# Add symlinks for pypy2 and pypy3 to usr/local/bin, there should only be 2 versions of PyPy in the tools cache that is downloaded
-	if [ ${pypy:0:1} -eq "3" ] ; then
-		# add pypy 3.X to PATH
-		ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy3 /usr/local/bin/pypy3
-	else
-		# add pypy 2.X to PATH
-		ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy /usr/local/bin/pypy
-	fi
-done;
-
-DocumentInstalledItem "Ruby:"
-rubys=$(ls $AGENT_TOOLSDIRECTORY/Ruby)
-for ruby in $rubys; do
-	DocumentInstalledItemIndent "Ruby $ruby"
-done;

images/linux/scripts/installers/boost.sh

@@ -7,7 +7,6 @@
 # Source the helpers for use with the script
 source $HELPER_SCRIPTS/document.sh
 
-BOOST_ZIP_PATH=/opt/hostedtoolcache/Boost
 BOOST_LIB=/usr/local/share/boost
 
 # Install Boost
@@ -15,9 +14,6 @@ for BOOST_VERSION in ${BOOST_VERSIONS//,/ }
 do
     BOOST_SYMLINK_VER=`echo "${BOOST_VERSION//[.]/_}"`
     BOOST_ROOT="BOOST_ROOT_$BOOST_SYMLINK_VER"
-    BOOST_ZIP="boost_`echo $BOOST_VERSION`_gcc.zip"
-
-    unzip $BOOST_ZIP_PATH/$BOOST_ZIP -d $BOOST_LIB
 
     echo "$BOOST_ROOT=$BOOST_LIB/$BOOST_VERSION" | tee -a /etc/environment
     if [[ $BOOST_VERSION == $BOOST_DEFAULT ]]; then
@@ -26,6 +22,3 @@ do
 
     DocumentInstalledItem "Boost C++ Libraries $BOOST_VERSION"
 done
-
-# Deleting archives with Boost Libraries
-rm -rf $BOOST_ZIP_PATH

images/linux/scripts/installers/hosted-tool-cache.sh

@@ -10,21 +10,23 @@ source $HELPER_SCRIPTS/document.sh
 # Fail out if any setups fail
 set -e
 
-# Download hosted tool cache
 AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache
+mkdir $AGENT_TOOLSDIRECTORY
 echo "AGENT_TOOLSDIRECTORY=$AGENT_TOOLSDIRECTORY" | tee -a /etc/environment
-azcopy --recursive \
-       --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux \
-       --destination $AGENT_TOOLSDIRECTORY
 
-# Install tools from hosted tool cache
-original_directory=$PWD
-setups=$(find $AGENT_TOOLSDIRECTORY -name setup.sh)
-for setup in $setups; do
-	chmod +x $setup;
-	cd $(dirname $setup);
-	./$(basename $setup);
-	cd $original_directory;
+chmod -R 777 $AGENT_TOOLSDIRECTORY
+
+echo "Installing npm-toolcache..."
+TOOLSET_PATH="$INSTALLER_SCRIPT_FOLDER/toolcache.json"
+
+PACKAGE_LIST=($(jq -r 'keys | .[]' $TOOLSET_PATH))
+
+for PACKAGE_NAME in ${PACKAGE_LIST[@]}; do
+    PACKAGE_VERSIONS=($(jq -r ".[\"$PACKAGE_NAME\"] | .[]" $TOOLSET_PATH))
+    for PACKAGE_VERSION in ${PACKAGE_VERSIONS[@]}; do
+        echo "Install ${PACKAGE_NAME}@${PACKAGE_VERSION}"
+        npm install ${PACKAGE_NAME}@${PACKAGE_VERSION} --registry=${TOOLCACHE_REGISTRY}
+    done;
 done;
 
 DocumentInstalledItem "Python:"
@@ -33,22 +35,14 @@ for python in $pythons; do
 	DocumentInstalledItemIndent "Python $python"
 done;
 
-# PyPy is also configured using the setup-python action
-pypys=$(ls $AGENT_TOOLSDIRECTORY/PyPy)
-for pypy in $pypys; do
-	DocumentInstalledItemIndent "PyPy $pypy"
-	# Add symlinks for pypy2 and pypy3 to usr/local/bin, there should only be 2 versions of PyPy in the tools cache that is downloaded
-	if [ ${pypy:0:1} -eq "3" ] ; then
-		# add pypy 3.X to PATH
-		ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy3 /usr/local/bin/pypy3
-	else
-		# add pypy 2.X to PATH
-		ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy /usr/local/bin/pypy
-	fi
-done;
-
 DocumentInstalledItem "Ruby:"
 rubys=$(ls $AGENT_TOOLSDIRECTORY/Ruby)
 for ruby in $rubys; do
 	DocumentInstalledItemIndent "Ruby $ruby"
-done;
+done;
+
+DocumentInstalledItem "PyPy:"
+pypys=$(ls $AGENT_TOOLSDIRECTORY/PyPy)
+for pypy in $pypys; do
+	DocumentInstalledItemIndent "PyPy $pypy"
+done;

images/linux/scripts/installers/test-toolcache.sh

@@ -9,6 +9,22 @@
 # Fail out if any tests fail
 set -e
 
+# defune array of key aliases
+key_alias_array=()
+bash --version
+# define dicionary for key_alias and its values
+declare -A toolcache_key_value
+PACKAGE_LIST=($(jq -r 'keys | .[]' "$INSTALLER_SCRIPT_FOLDER/toolcache.json"))
+for PACKAGE_NAME in ${PACKAGE_LIST[@]}; do
+
+   # get key alias
+   key_alias=$(echo $PACKAGE_NAME | cut -f2 -d-)
+   echo $key_alias
+
+   # set dictionary
+   toolcache_key_value+=(["$key_alias"]="$PACKAGE_NAME")
+done;
+
 AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache
 
 # Python test
@@ -18,19 +34,19 @@ if [ -d "$AGENT_TOOLSDIRECTORY/Python" ]; then
    echo "Python versions folders: ${python_dirs[@]}"
    echo "------------------------------------------"
    if [ -n "$python_dirs" ]; then
-      for version_dir in "${python_dirs[@]}"
+      PACKAGE_VERSIONS=($(jq -r ".[\"${toolcache_key_value[python]}\"] | .[]" "$INSTALLER_SCRIPT_FOLDER/toolcache.json"))
+      for python_version in ${PACKAGE_VERSIONS[@]}
       do
+         version_dir=$(find . -name "$python_version.*" -print -quit)
          echo "Test $AGENT_TOOLSDIRECTORY/Python/$version_dir:"
-         expected_ver=$(echo $version_dir | egrep -o '[0-9]+\.[0-9]+')
          actual_ver=$($AGENT_TOOLSDIRECTORY/Python/$version_dir/x64/python -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+')
-
-         if [ "$expected_ver" = "$actual_ver" ]; then
+         if [ "$actual_ver" = "$python_version" ]; then
                echo "Passed!"
          else
-               echo "Expected: $expected_ver; Actual: $actual_ver"
+               echo "Expected: $python_version; Actual: $actual_ver"
                exit 1
          fi
-      done
+      done;
    else
       echo "$AGENT_TOOLSDIRECTORY/Python does not include any folders"
       exit 1
@@ -47,19 +63,19 @@ if [ -d "$AGENT_TOOLSDIRECTORY/Ruby" ]; then
    echo "Ruby versions folders: ${ruby_dirs[@]}"
    echo "--------------------------------------"
    if [ -n "$ruby_dirs" ]; then
-      for version_dir in "${ruby_dirs[@]}"
+      PACKAGE_VERSIONS=($(jq -r ".[\"${toolcache_key_value[ruby]}\"] | .[]" "$INSTALLER_SCRIPT_FOLDER/toolcache.json"))
+      for ruby_version in ${PACKAGE_VERSIONS[@]}
       do
+         version_dir=$(find . -name "$ruby_version.*" -print -quit)
          echo "Test $AGENT_TOOLSDIRECTORY/Ruby/$version_dir:"
-         expected_ver=$(echo $version_dir | egrep -o '[0-9]+\.[0-9]+')
          actual_ver=$($AGENT_TOOLSDIRECTORY/Ruby/$version_dir/x64/bin/ruby -e "puts RUBY_VERSION" | egrep -o '[0-9]+\.[0-9]+')
-
-         if [ "$expected_ver" = "$actual_ver" ]; then
-            echo "Passed!"
+         if [ "$actual_ver" = "$ruby_version" ]; then
+               echo "Passed!"
          else
-            echo "Expected: $expected_ver; Actual: $actual_ver"
-            exit 1
+               echo "Expected: $ruby_version; Actual: $actual_ver"
+               exit 1
          fi
-      done
+      done;
    else
       echo "$AGENT_TOOLSDIRECTORY/Ruby does not include any folders"
       exit 1
@@ -68,3 +84,34 @@ else
    echo "$AGENT_TOOLSDIRECTORY/Ruby does not exist"
    exit 1
 fi
+
+# PyPy tests
+if [ -d "$AGENT_TOOLSDIRECTORY/PyPy" ]; then
+   cd $AGENT_TOOLSDIRECTORY/PyPy
+   pypy_dirs=($(find . -mindepth 1 -maxdepth 1 -type d | sed "s|^\./||"))
+   echo "PyPy versions folders: ${pypy_dirs[@]}"
+   echo "------------------------------------------"
+   if [ -n "$pypy_dirs" ]; then
+      PACKAGE_VERSIONS=($(jq -r ".[\"${toolcache_key_value[pypy]}\"] | .[]" "$INSTALLER_SCRIPT_FOLDER/toolcache.json"))
+      for pypy_version in ${PACKAGE_VERSIONS[@]}
+      do
+         version_dir=$(find . -name "$pypy_version.*" -print -quit)
+         pypy_path=$(find . -regex "$version_dir/x64/bin/pypy[0-9]*" -print -quit)
+         echo $pypy_path
+         actual_ver=$($pypy_path -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+' | cut -d '.' -f 1)
+         echo "actual_ver = $actual_ver : pypy_version = $pypy_version"
+         if [ "$actual_ver" = "$pypy_version" ]; then
+               echo "Passed!"
+         else
+               echo "Expected: $pypy_version; Actual: $actual_ver"
+               exit 1
+         fi
+      done;
+   else
+      echo "$AGENT_TOOLSDIRECTORY/PyPy does not include any folders"
+      exit 1
+   fi
+else
+   echo "$AGENT_TOOLSDIRECTORY/PyPy does not exist"
+   exit 1
+fi