Merge pull request #216 from vmapetr/v-mapetr/add-npm-toolcache

Toolcache: Add NPM Toolcache support for Windows/Linux

images/linux/scripts/installers/1804/hosted-tool-cache.sh

@@ -1,54 +0,0 @@
-#!/bin/bash
-################################################################################
-##  File:  hosted-tool-cache.sh
-##  Desc:  Downloads and installs hosted tools cache
-################################################################################
-
-# Source the helpers for use with the script
-source $HELPER_SCRIPTS/document.sh
-
-# Fail out if any setups fail
-set -e
-
-# Download hosted tool cache
-AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache
-echo "AGENT_TOOLSDIRECTORY=$AGENT_TOOLSDIRECTORY" | tee -a /etc/environment
-azcopy --recursive \
-       --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/ubuntu-1804 \
-       --destination $AGENT_TOOLSDIRECTORY
-
-# Install tools from hosted tool cache
-original_directory=$PWD
-setups=$(find $AGENT_TOOLSDIRECTORY -name setup.sh)
-for setup in $setups; do
-	chmod +x $setup;
-	cd $(dirname $setup);
-	./$(basename $setup);
-	cd $original_directory;
-done;
-
-DocumentInstalledItem "Python:"
-pythons=$(ls $AGENT_TOOLSDIRECTORY/Python)
-for python in $pythons; do
-	DocumentInstalledItemIndent "Python $python"
-done;
-
-# PyPy is also configured using the setup-python action
-pypys=$(ls $AGENT_TOOLSDIRECTORY/PyPy)
-for pypy in $pypys; do
-	DocumentInstalledItemIndent "PyPy $pypy"
-	# Add symlinks for pypy2 and pypy3 to usr/local/bin, there should only be 2 versions of PyPy in the tools cache that is downloaded
-	if [ ${pypy:0:1} -eq "3" ] ; then
-		# add pypy 3.X to PATH
-		ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy3 /usr/local/bin/pypy3
-	else
-		# add pypy 2.X to PATH
-		ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy /usr/local/bin/pypy
-	fi
-done;
-
-DocumentInstalledItem "Ruby:"
-rubys=$(ls $AGENT_TOOLSDIRECTORY/Ruby)
-for ruby in $rubys; do
-	DocumentInstalledItemIndent "Ruby $ruby"
-done;

images/linux/scripts/installers/boost.sh

@@ -7,7 +7,6 @@
 # Source the helpers for use with the script
 source $HELPER_SCRIPTS/document.sh
 
-BOOST_ZIP_PATH=/opt/hostedtoolcache/Boost
 BOOST_LIB=/usr/local/share/boost
 
 # Install Boost
@@ -15,9 +14,6 @@ for BOOST_VERSION in ${BOOST_VERSIONS//,/ }
 do
     BOOST_SYMLINK_VER=`echo "${BOOST_VERSION//[.]/_}"`
     BOOST_ROOT="BOOST_ROOT_$BOOST_SYMLINK_VER"
-    BOOST_ZIP="boost_`echo $BOOST_VERSION`_gcc.zip"
-
-    unzip $BOOST_ZIP_PATH/$BOOST_ZIP -d $BOOST_LIB
 
     echo "$BOOST_ROOT=$BOOST_LIB/$BOOST_VERSION" | tee -a /etc/environment
     if [[ $BOOST_VERSION == $BOOST_DEFAULT ]]; then
@@ -26,6 +22,3 @@ do
 
     DocumentInstalledItem "Boost C++ Libraries $BOOST_VERSION"
 done
-
-# Deleting archives with Boost Libraries
-rm -rf $BOOST_ZIP_PATH

images/linux/scripts/installers/hosted-tool-cache.sh

@@ -10,21 +10,23 @@ source $HELPER_SCRIPTS/document.sh
 # Fail out if any setups fail
 set -e
 
-# Download hosted tool cache
 AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache
+mkdir $AGENT_TOOLSDIRECTORY
 echo "AGENT_TOOLSDIRECTORY=$AGENT_TOOLSDIRECTORY" | tee -a /etc/environment
-azcopy --recursive \
-       --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux \
-       --destination $AGENT_TOOLSDIRECTORY
 
-# Install tools from hosted tool cache
+chmod -R 777 $AGENT_TOOLSDIRECTORY
-original_directory=$PWD
+
-setups=$(find $AGENT_TOOLSDIRECTORY -name setup.sh)
+echo "Installing npm-toolcache..."
-for setup in $setups; do
+TOOLSET_PATH="$INSTALLER_SCRIPT_FOLDER/toolcache.json"
-	chmod +x $setup;
+
-	cd $(dirname $setup);
+PACKAGE_LIST=($(jq -r 'keys | .[]' $TOOLSET_PATH))
-	./$(basename $setup);
+
-	cd $original_directory;
+for PACKAGE_NAME in ${PACKAGE_LIST[@]}; do
+    PACKAGE_VERSIONS=($(jq -r ".[\"$PACKAGE_NAME\"] | .[]" $TOOLSET_PATH))
+    for PACKAGE_VERSION in ${PACKAGE_VERSIONS[@]}; do
+        echo "Install ${PACKAGE_NAME}@${PACKAGE_VERSION}"
+        npm install ${PACKAGE_NAME}@${PACKAGE_VERSION} --registry=${TOOLCACHE_REGISTRY}
+    done;
 done;
 
 DocumentInstalledItem "Python:"
@@ -33,22 +35,14 @@ for python in $pythons; do
 	DocumentInstalledItemIndent "Python $python"
 done;
 
-# PyPy is also configured using the setup-python action
-pypys=$(ls $AGENT_TOOLSDIRECTORY/PyPy)
-for pypy in $pypys; do
-	DocumentInstalledItemIndent "PyPy $pypy"
-	# Add symlinks for pypy2 and pypy3 to usr/local/bin, there should only be 2 versions of PyPy in the tools cache that is downloaded
-	if [ ${pypy:0:1} -eq "3" ] ; then
-		# add pypy 3.X to PATH
-		ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy3 /usr/local/bin/pypy3
-	else
-		# add pypy 2.X to PATH
-		ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy /usr/local/bin/pypy
-	fi
-done;
-
 DocumentInstalledItem "Ruby:"
 rubys=$(ls $AGENT_TOOLSDIRECTORY/Ruby)
 for ruby in $rubys; do
 	DocumentInstalledItemIndent "Ruby $ruby"
-done;
+done;
+
+DocumentInstalledItem "PyPy:"
+pypys=$(ls $AGENT_TOOLSDIRECTORY/PyPy)
+for pypy in $pypys; do
+	DocumentInstalledItemIndent "PyPy $pypy"
+done;

images/linux/scripts/installers/test-toolcache.sh

@@ -6,65 +6,69 @@
 
 # Must be procecessed after tool cache setup(hosted-tool-cache.sh).
 
+Test_Hostedtoolcache_Tool() {
+   TOOL_NAME=$1
+   TOOL_EXEC_TEST=$2
+
+   if [ -d "$AGENT_TOOLSDIRECTORY/$TOOL_NAME" ]; then
+      cd $AGENT_TOOLSDIRECTORY/$TOOL_NAME
+
+      tool_dirs=($(find . -mindepth 1 -maxdepth 1 -type d | sed "s|^\./||"))
+
+      echo "--------------------------------------------"
+      echo "$TOOL_NAME versions folders: ${tool_dirs[@]}"
+      echo "--------------------------------------------"
+
+      if [ -n "$tool_dirs" ]; then
+         tool_key=$(echo $TOOL_NAME | tr "[:upper:]" "[:lower:]")
+         package_versions=($(jq -r ".[\"${TOOLCACHE_KEY_VALUE[$tool_key]}\"] | .[]" "$INSTALLER_SCRIPT_FOLDER/toolcache.json"))
+
+         for tool_version in ${package_versions[@]}
+         do
+            version_dir=$(find . -name "$tool_version.*" -print -quit)
+
+            echo "Test $AGENT_TOOLSDIRECTORY/$TOOL_NAME/$version_dir:"
+
+            actual_version=$(eval $AGENT_TOOLSDIRECTORY/$TOOL_NAME/$version_dir/$TOOL_EXEC_TEST)
+
+            if [ "$actual_version" = "$tool_version" ]; then
+                  echo "Passed!"
+            else
+                  echo "Expected: $tool_version; Actual: $actual_version"
+
+                  exit 1
+            fi
+         done;
+      else
+         echo "$AGENT_TOOLSDIRECTORY/$tool_version does not include any folders"
+
+         exit 1
+      fi
+   else
+      echo "$AGENT_TOOLSDIRECTORY/$tool_version does not exist"
+
+      exit 1
+   fi
+}
+
 # Fail out if any tests fail
 set -e
 
+# define dictionary for key_alias and its values
+declare -A TOOLCACHE_KEY_VALUE
+
+package_list=($(jq -r 'keys | .[]' "$INSTALLER_SCRIPT_FOLDER/toolcache.json"))
+
+for package_name in ${package_list[@]}; do
+   # get key alias
+   key_alias=$(echo $package_name | cut -f2 -d-)
+
+   # set dictionary
+   TOOLCACHE_KEY_VALUE+=(["$key_alias"]="$package_name")
+done;
+
 AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache
 
-# Python test
+Test_Hostedtoolcache_Tool "Python" "x64/python -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+'"
-if [ -d "$AGENT_TOOLSDIRECTORY/Python" ]; then
+Test_Hostedtoolcache_Tool "Ruby" "x64/bin/ruby -e 'puts RUBY_VERSION' | egrep -o '[0-9]+\.[0-9]+'"
-   cd $AGENT_TOOLSDIRECTORY/Python
+Test_Hostedtoolcache_Tool "PyPy" "x64/bin/python -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+' | cut -d '.' -f 1"
-   python_dirs=($(find . -mindepth 1 -maxdepth 1 -type d | sed "s|^\./||"))
-   echo "Python versions folders: ${python_dirs[@]}"
-   echo "------------------------------------------"
-   if [ -n "$python_dirs" ]; then
-      for version_dir in "${python_dirs[@]}"
-      do
-         echo "Test $AGENT_TOOLSDIRECTORY/Python/$version_dir:"
-         expected_ver=$(echo $version_dir | egrep -o '[0-9]+\.[0-9]+')
-         actual_ver=$($AGENT_TOOLSDIRECTORY/Python/$version_dir/x64/python -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+')
-
-         if [ "$expected_ver" = "$actual_ver" ]; then
-               echo "Passed!"
-         else
-               echo "Expected: $expected_ver; Actual: $actual_ver"
-               exit 1
-         fi
-      done
-   else
-      echo "$AGENT_TOOLSDIRECTORY/Python does not include any folders"
-      exit 1
-   fi
-else
-   echo "$AGENT_TOOLSDIRECTORY/Python does not exist"
-   exit 1
-fi
-
-# Ruby test
-if [ -d "$AGENT_TOOLSDIRECTORY/Ruby" ]; then
-   cd $AGENT_TOOLSDIRECTORY/Ruby
-   ruby_dirs=($(find . -mindepth 1 -maxdepth 1 -type d | sed "s|^\./||"))
-   echo "Ruby versions folders: ${ruby_dirs[@]}"
-   echo "--------------------------------------"
-   if [ -n "$ruby_dirs" ]; then
-      for version_dir in "${ruby_dirs[@]}"
-      do
-         echo "Test $AGENT_TOOLSDIRECTORY/Ruby/$version_dir:"
-         expected_ver=$(echo $version_dir | egrep -o '[0-9]+\.[0-9]+')
-         actual_ver=$($AGENT_TOOLSDIRECTORY/Ruby/$version_dir/x64/bin/ruby -e "puts RUBY_VERSION" | egrep -o '[0-9]+\.[0-9]+')
-
-         if [ "$expected_ver" = "$actual_ver" ]; then
-            echo "Passed!"
-         else
-            echo "Expected: $expected_ver; Actual: $actual_ver"
-            exit 1
-         fi
-      done
-   else
-      echo "$AGENT_TOOLSDIRECTORY/Ruby does not include any folders"
-      exit 1
-   fi
-else
-   echo "$AGENT_TOOLSDIRECTORY/Ruby does not exist"
-   exit 1
-fi

images/linux/toolcache-1604.json

@@ -0,0 +1,14 @@
+{
+    "toolcache-python-ubuntu-1604-x64": [
+        "2.7", "3.5", "3.6", "3.7", "3.8"
+    ],
+    "toolcache-ruby-ubuntu-1604-x64": [
+        "2.4", "2.5", "2.6", "2.7"
+    ],
+    "toolcache-pypy-ubuntu-1604-x64": [
+        "2", "3"
+    ],
+    "toolcache-boost-ubuntu-1604-x64": [
+        "1.69"
+    ]
+}

images/linux/toolcache-1804.json

@@ -0,0 +1,14 @@
+{
+    "toolcache-python-ubuntu-1804-x64": [
+        "2.7", "3.5", "3.6", "3.7", "3.8"
+    ],
+    "toolcache-ruby-ubuntu-1804-x64": [
+        "2.4", "2.5", "2.6", "2.7"
+    ],
+    "toolcache-pypy-ubuntu-1804-x64": [
+        "2", "3"
+    ],
+    "toolcache-boost-ubuntu-1804-x64": [
+        "1.69"
+    ]
+}

images/linux/ubuntu1604.json

@@ -20,7 +20,8 @@
         "helper_script_folder": "/imagegeneration/helpers",
         "vm_size": "Standard_DS2_v2",
         "capture_name_prefix": "packer",
-        "image_version": "dev"
+        "image_version": "dev",
+        "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/"
     },
     "sensitive-variables": ["client_secret"],
     "builders": [
@@ -168,19 +169,26 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolcache-1604.json",
+            "destination": "{{user `installer_script_folder`}}/toolcache.json"
+        },
         {
             "type": "shell",
             "scripts":[
                 "{{template_dir}}/scripts/installers/1604/android.sh",
                 "{{template_dir}}/scripts/installers/1604/azpowershell.sh",
                 "{{template_dir}}/scripts/helpers/containercache.sh",
-                "{{template_dir}}/scripts/installers/1604/hosted-tool-cache.sh",
+                "{{template_dir}}/scripts/installers/hosted-tool-cache.sh",
                 "{{template_dir}}/scripts/installers/python.sh",
                 "{{template_dir}}/scripts/installers/test-toolcache.sh"
             ],
             "environment_vars": [
                 "METADATA_FILE={{user `metadata_file`}}",
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}"
+                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
+                "TOOLCACHE_REGISTRY={{user `toolcache_registry`}}",
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },

images/linux/ubuntu1804.json

@@ -20,7 +20,8 @@
         "helper_script_folder": "/imagegeneration/helpers",
         "vm_size": "Standard_DS2_v2",
         "capture_name_prefix": "packer",
-        "image_version": "dev"
+        "image_version": "dev",
+        "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/"
     },
     "sensitive-variables": ["client_secret"],
     "builders": [
@@ -171,19 +172,26 @@
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolcache-1804.json",
+            "destination": "{{user `installer_script_folder`}}/toolcache.json"
+        },
         {
             "type": "shell",
             "scripts":[
                 "{{template_dir}}/scripts/installers/1804/android.sh",
                 "{{template_dir}}/scripts/installers/1804/azpowershell.sh",
                 "{{template_dir}}/scripts/helpers/containercache.sh",
-                "{{template_dir}}/scripts/installers/1804/hosted-tool-cache.sh",
+                "{{template_dir}}/scripts/installers/hosted-tool-cache.sh",
                 "{{template_dir}}/scripts/installers/python.sh",
                 "{{template_dir}}/scripts/installers/test-toolcache.sh"
             ],
             "environment_vars": [
                 "METADATA_FILE={{user `metadata_file`}}",
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}"
+                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
+                "TOOLCACHE_REGISTRY={{user `toolcache_registry`}}",
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },

images/win/Windows2016-Azure.json

@@ -16,6 +16,7 @@
         "private_virtual_network_with_public_ip": "{{env `PRIVATE_VIRTUAL_NETWORK_WITH_PUBLIC_IP`}}",
         "vm_size": "Standard_DS4_v2",
 
+        "root_folder": "C:",
         "image_folder": "C:\\image",
         "commit_file": "C:\\image\\commit.txt",
         "metadata_file": "C:\\image\\metadata.txt",
@@ -24,7 +25,8 @@
         "install_user": "installer",
         "install_password": null,
         "capture_name_prefix": "packer",
-        "image_version": "dev"
+        "image_version": "dev",
+        "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/"
     },
     "sensitive-variables": ["install_password", "ssh_password", "client_secret"],
     "builders": [
@@ -269,6 +271,27 @@
         },
         {
             "type": "powershell",
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Install-NodeLts.ps1"
+            ]
+        },
+        {
+            "type": "powershell",
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Install-7zip.ps1"
+            ]
+        },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolcache.json",
+            "destination": "{{user `root_folder`}}/toolcache.json"
+        },
+        {
+            "type": "powershell",
+            "environment_vars":[
+                "TOOLCACHE_REGISTRY={{ user `toolcache_registry` }}",
+                "ROOT_FOLDER={{user `root_folder`}}"
+            ],
             "scripts":[
                 "{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1"
             ]
@@ -369,12 +392,6 @@
                 "{{ template_dir }}/scripts/Installers/Validate-WinAppDriver.ps1"
             ]
         },
-        {
-            "type": "powershell",
-            "scripts":[
-                "{{ template_dir }}/scripts/Installers/Install-NodeLts.ps1"
-            ]
-        },
         {
             "type": "powershell",
             "scripts":[
@@ -457,12 +474,6 @@
                 "{{ template_dir }}/scripts/Installers/Install-AzureCosmosDbEmulator.ps1"
             ]
         },
-        {
-            "type": "powershell",
-            "scripts":[
-                "{{ template_dir }}/scripts/Installers/Install-7zip.ps1"
-            ]
-        },
         {
             "type": "powershell",
             "environment_vars": [
@@ -569,6 +580,9 @@
         },
         {
             "type": "powershell",
+            "environment_vars":[
+                "ROOT_FOLDER={{user `root_folder`}}"
+            ],
             "scripts":[
                 "{{ template_dir }}/scripts/Installers/Validate-ToolCache.ps1"
             ]

images/win/Windows2019-Azure.json

@@ -16,6 +16,7 @@
         "private_virtual_network_with_public_ip": "{{env `PRIVATE_VIRTUAL_NETWORK_WITH_PUBLIC_IP`}}",
         "vm_size": "Standard_DS4_v2",
 
+        "root_folder": "C:",
         "image_folder": "C:\\image",
         "commit_file": "C:\\image\\commit.txt",
         "metadata_file": "C:\\image\\metadata.txt",
@@ -24,7 +25,8 @@
         "install_user": "installer",
         "install_password": null,
         "capture_name_prefix": "packer",
-        "image_version": "dev"
+        "image_version": "dev",
+        "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/"
     },
     "sensitive-variables": ["install_password", "ssh_password", "client_secret"],
     "builders": [
@@ -238,6 +240,27 @@
         },
         {
             "type": "powershell",
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Install-NodeLts.ps1"
+            ]
+        },
+        {
+            "type": "powershell",
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Install-7zip.ps1"
+            ]
+        },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolcache.json",
+            "destination": "{{user `root_folder`}}/toolcache.json"
+        },
+        {
+            "type": "powershell",
+            "environment_vars":[
+                "TOOLCACHE_REGISTRY={{ user `toolcache_registry` }}",
+                "ROOT_FOLDER={{user `root_folder`}}"
+            ],
             "scripts":[
                 "{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1"
             ]
@@ -338,12 +361,6 @@
                 "{{ template_dir }}/scripts/Installers/Validate-WinAppDriver.ps1"
             ]
         },
-        {
-            "type": "powershell",
-            "scripts":[
-                "{{ template_dir }}/scripts/Installers/Install-NodeLts.ps1"
-            ]
-        },
         {
             "type": "powershell",
             "scripts":[
@@ -426,12 +443,6 @@
                 "{{ template_dir }}/scripts/Installers/Install-AzureCosmosDbEmulator.ps1"
             ]
         },
-        {
-            "type": "powershell",
-            "scripts":[
-                "{{ template_dir }}/scripts/Installers/Install-7zip.ps1"
-            ]
-        },
         {
             "type": "powershell",
             "environment_vars": [
@@ -538,6 +549,9 @@
         },
         {
             "type": "powershell",
+            "environment_vars":[
+                "ROOT_FOLDER={{user `root_folder`}}"
+            ],
             "scripts":[
                 "{{ template_dir }}/scripts/Installers/Validate-ToolCache.ps1"
             ]

images/win/scripts/Installers/Download-ToolCache.ps1

@@ -1,51 +1,48 @@
 ################################################################################
 ##  File:  Download-ToolCache.ps1
+##  Team:  CI-Build
 ##  Desc:  Download tool cache
 ################################################################################
 
-Function InstallTool
+Function Install-NpmPackage {
-{
+    [CmdletBinding()]
-    Param
+    param(
-    (
+        [Parameter(Mandatory=$true)]
-        [System.Object]$ExecutablePath
+        [System.String]
+        $Name,
+        [Parameter(Mandatory=$true)]
+        [System.String]
+        $NpmRegistry
     )
 
-    Write-Host $ExecutablePath.DirectoryName
+    Write-Host "Installing npm '$Name' package from '$NpmRegistry'"
-    Set-Location -Path $ExecutablePath.DirectoryName
+
-    Get-Location | Write-Host
+    npm install $Name --registry=$NpmRegistry
-    if (Test-Path 'tool.zip')
-    {
-        Expand-Archive 'tool.zip' -DestinationPath '.'
-    }
-    cmd.exe /c 'install_to_tools_cache.bat'
 }
 
-$SourceUrl = "https://vstsagenttools.blob.core.windows.net/tools"
+# HostedToolCache Path
-
 $Dest = "C:/"
-
 $Path = "hostedtoolcache/windows"
-
-$env:Path = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy;" + $env:Path
-
-Write-Host "Started AzCopy from $SourceUrl to $Dest"
-
-AzCopy /Source:$SourceUrl /Dest:$Dest  /S /V /Pattern:$Path
-
 $ToolsDirectory = $Dest + $Path
 
-$current = Get-Location
+# Define AGENT_TOOLSDIRECTORY environment variable
-Set-Location -Path $ToolsDirectory
+$env:AGENT_TOOLSDIRECTORY = $ToolsDirectory
-
-Get-ChildItem -Recurse -Depth 4 -Filter install_to_tools_cache.bat | ForEach-Object {
-    #In order to work correctly Python 3.4 x86 must be installed after x64, this is achieved by current toolcache catalog structure
-    InstallTool($_)
-}
-
-Set-Location -Path $current
-
 setx AGENT_TOOLSDIRECTORY $ToolsDirectory /M
 
+# Install HostedToolCache tools via NPM
+$ToolVersionsFileContent = Get-Content -Path "$env:ROOT_FOLDER/toolcache.json" -Raw
+$ToolVersions = ConvertFrom-Json -InputObject $ToolVersionsFileContent
+
+$ToolVersions.PSObject.Properties | ForEach-Object {
+    $PackageName = $_.Name
+    $PackageVersions = $_.Value
+    $NpmPackages = $PackageVersions | ForEach-Object { "$PackageName@$_" }
+    foreach($NpmPackage in $NpmPackages) {
+        Install-NpmPackage -Name $NpmPackage -NpmRegistry $env:TOOLCACHE_REGISTRY
+    }
+}
+
 #junction point from the previous Python2 directory to the toolcache Python2
+Write-Host "Create symlink to Python2"
 $python2Dir = (Get-Item -Path ($ToolsDirectory + '/Python/2.7*/x64')).FullName
 cmd.exe /c mklink /d "C:\Python27amd64" "$python2Dir"

images/win/scripts/Installers/Install-Boost.ps1

@@ -1,28 +1,16 @@
 ################################################################################
-##  File:  Install-Boost.ps1
+##  File:  Set-BoostRoot.ps1
+##  Team:  CI-Build
 ##  Desc:  Install boost using tool cache
 ################################################################################
 
-$ToolCache = 'C:\hostedtoolcache\windows\boost'
 $BoostDirectory = Join-Path -Path $env:ProgramFiles -ChildPath "Boost"
-$BoostVersionsToInstall = $env:BOOST_VERSIONS.split(',')
+$BoostVersions = $env:BOOST_VERSIONS.split(',')
 $BoostDefault = $env:BOOST_DEFAULT
 
-foreach($BoostVersion in $BoostVersionsToInstall)
+foreach($BoostVersion in $BoostVersions)
 {
-    $ZipName = Join-Path -Path $ToolCache -ChildPath "boost_${BoostVersion}_msvc-14.1.zip"
-
-    if (-Not (Test-Path $ZipName))
-    {
-        Write-Host "$ZipName not found"
-        exit 1
-    }
-    
-    Write-Host "Expanding $ZipName"
-
     $BoostInstallationDir = Join-Path -Path $BoostDirectory -ChildPath $BoostVersion
-    # Expand-Archive slower for 70% than 7z
-    & "$env:ProgramFiles\7-Zip\7z.exe" x $ZipName -o"$BoostDirectory" -y
 
     $EnvBoostPath = "BOOST_ROOT_{0}" -f ($BoostVersion.Replace('.', '_'))
     setx $EnvBoostPath $BoostInstallationDir /M | Out-Null
@@ -36,10 +24,3 @@ foreach($BoostVersion in $BoostVersionsToInstall)
         setx BOOST_ROOT $BoostInstallationDir /M | Out-Null
     }
 }
-
-# Removing boost cache folder
-Write-Host "Removing boost cache folder"
-if (Test-Path $ToolCache)
-{
-    Remove-Item -Path $ToolCache -Force -Recurse
-}

images/win/scripts/Installers/Validate-ToolCache.ps1

@@ -12,6 +12,21 @@ function GetChildFolders {
     return Get-ChildItem -Path $Path -Directory -Name
 }
 
+function Get-ToolcachePackages {
+    $toolcachePath = Join-Path $env:ROOT_FOLDER "toolcache.json"
+    return Get-Content -Raw $toolcachePath | ConvertFrom-Json
+}
+
+$packages = (Get-ToolcachePackages).PSObject.Properties | ForEach-Object {
+    $packageNameParts = $_.Name.Split("-")
+    $toolName = $packageNameParts[1]
+    return [PSCustomObject] @{
+        ToolName = $packageNameParts[1]
+        Versions = $_.Value
+        Arch = $packageNameParts[3]
+    }
+}
+
 function ToolcacheTest {
     param (
         [Parameter(Mandatory = $True)]
@@ -22,36 +37,53 @@ function ToolcacheTest {
     if (Test-Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName")
     {
         $description = ""
-        [array]$versions = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName"
+        [array]$installedVersions = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName"
-        if ($versions.count -gt 0){
+        if ($installedVersions.count -gt 0){
-            foreach ($version in $versions)
+            $softwarePackages = $packages | Where-Object { $_.ToolName -eq $SoftwareName }
+            foreach($softwarePackage in $softwarePackages)
             {
-                $architectures = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$version"
+                foreach ($version in $softwarePackage.Versions)
-
-                Write-Host "$SoftwareName version - $version : $([system.String]::Join(",", $architectures))"
-
-                foreach ($arch in $architectures)
                 {
-                    $path = "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$version\$arch"
+                    $foundVersion = $installedVersions | where { $_.StartsWith($version) }
-                    foreach ($test in $ExecTests)
+
-                    {
+                    if ($foundVersion -ne $null){
-                        if (Test-Path "$path\$test")
+
-                        {
+                        $architectures = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$foundVersion"
-                            Write-Host "$SoftwareName($test) $version($arch) is successfully installed:"
+
-                            Write-Host (& "$path\$test" --version)
+                        $softwareArch = $softwarePackage.Arch
+
+                        if ($architectures -Contains $softwareArch) {
+                            $path = "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$foundVersion\$softwareArch"
+                            foreach ($test in $ExecTests)
+                            {
+                                if (Test-Path "$path\$test")
+                                {
+                                    Write-Host "$SoftwareName($test) $foundVersion($softwareArch) is successfully installed:"
+                                    Write-Host (& "$path\$test" --version)
+                                }
+                                else
+                                {
+                                    Write-Host "$SoftwareName($test) $foundVersion ($softwareArch) is not installed"
+                                    exit 1
+                                }
+                            }
+                            $description += "_Version:_ $foundVersion ($softwareArch)<br/>"
                         }
                         else
                         {
-                            Write-Host "$SoftwareName($test) $version ($arch) is not installed"
+                            Write-Host "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$foundVersion does not include required architecture"
                             exit 1
                         }
                     }
-
+                    else
-                    $description += "_Version:_ $version ($arch)<br/>"
+                    {
+                        Write-Host "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$version.* was not found"
+                        exit 1
+                    }
                 }
-            }
 
-            Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $description
+                Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $description
+            }
         }
         else
         {

images/win/toolcache.json

@@ -0,0 +1,17 @@
+{
+    "toolcache-python-windows-x64": [
+        "2.7", "3.5", "3.6", "3.7", "3.8"
+    ],
+    "toolcache-python-windows-x86": [
+        "2.7", "3.5", "3.6", "3.7", "3.8"
+    ],
+    "toolcache-ruby-windows-x64": [
+        "2.4", "2.5", "2.6", "2.7"
+    ],
+    "toolcache-pypy-windows-x86": [
+        "2", "3"
+    ],
+    "toolcache-boost-windows-x64": [
+        "1.69"
+    ]
+}