diff --git a/images/linux/scripts/installers/1804/hosted-tool-cache.sh b/images/linux/scripts/installers/1804/hosted-tool-cache.sh deleted file mode 100644 index 10180a240..000000000 --- a/images/linux/scripts/installers/1804/hosted-tool-cache.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -################################################################################ -## File: hosted-tool-cache.sh -## Desc: Downloads and installs hosted tools cache -################################################################################ - -# Source the helpers for use with the script -source $HELPER_SCRIPTS/document.sh - -# Fail out if any setups fail -set -e - -# Download hosted tool cache -AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache -echo "AGENT_TOOLSDIRECTORY=$AGENT_TOOLSDIRECTORY" | tee -a /etc/environment -azcopy --recursive \ - --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/ubuntu-1804 \ - --destination $AGENT_TOOLSDIRECTORY - -# Install tools from hosted tool cache -original_directory=$PWD -setups=$(find $AGENT_TOOLSDIRECTORY -name setup.sh) -for setup in $setups; do - chmod +x $setup; - cd $(dirname $setup); - ./$(basename $setup); - cd $original_directory; -done; - -DocumentInstalledItem "Python:" -pythons=$(ls $AGENT_TOOLSDIRECTORY/Python) -for python in $pythons; do - DocumentInstalledItemIndent "Python $python" -done; - -# PyPy is also configured using the setup-python action -pypys=$(ls $AGENT_TOOLSDIRECTORY/PyPy) -for pypy in $pypys; do - DocumentInstalledItemIndent "PyPy $pypy" - # Add symlinks for pypy2 and pypy3 to usr/local/bin, there should only be 2 versions of PyPy in the tools cache that is downloaded - if [ ${pypy:0:1} -eq "3" ] ; then - # add pypy 3.X to PATH - ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy3 /usr/local/bin/pypy3 - else - # add pypy 2.X to PATH - ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy /usr/local/bin/pypy - fi -done; - -DocumentInstalledItem "Ruby:" -rubys=$(ls $AGENT_TOOLSDIRECTORY/Ruby) -for ruby in $rubys; do - DocumentInstalledItemIndent "Ruby $ruby" -done; \ No newline at end of file diff --git a/images/linux/scripts/installers/boost.sh b/images/linux/scripts/installers/boost.sh index 3b7d6b043..673b1f61e 100644 --- a/images/linux/scripts/installers/boost.sh +++ b/images/linux/scripts/installers/boost.sh @@ -7,7 +7,6 @@ # Source the helpers for use with the script source $HELPER_SCRIPTS/document.sh -BOOST_ZIP_PATH=/opt/hostedtoolcache/Boost BOOST_LIB=/usr/local/share/boost # Install Boost @@ -15,9 +14,6 @@ for BOOST_VERSION in ${BOOST_VERSIONS//,/ } do BOOST_SYMLINK_VER=`echo "${BOOST_VERSION//[.]/_}"` BOOST_ROOT="BOOST_ROOT_$BOOST_SYMLINK_VER" - BOOST_ZIP="boost_`echo $BOOST_VERSION`_gcc.zip" - - unzip $BOOST_ZIP_PATH/$BOOST_ZIP -d $BOOST_LIB echo "$BOOST_ROOT=$BOOST_LIB/$BOOST_VERSION" | tee -a /etc/environment if [[ $BOOST_VERSION == $BOOST_DEFAULT ]]; then @@ -26,6 +22,3 @@ do DocumentInstalledItem "Boost C++ Libraries $BOOST_VERSION" done - -# Deleting archives with Boost Libraries -rm -rf $BOOST_ZIP_PATH diff --git a/images/linux/scripts/installers/1604/hosted-tool-cache.sh b/images/linux/scripts/installers/hosted-tool-cache.sh similarity index 51% rename from images/linux/scripts/installers/1604/hosted-tool-cache.sh rename to images/linux/scripts/installers/hosted-tool-cache.sh index c68f9747e..66ab532ea 100644 --- a/images/linux/scripts/installers/1604/hosted-tool-cache.sh +++ b/images/linux/scripts/installers/hosted-tool-cache.sh @@ -10,21 +10,23 @@ source $HELPER_SCRIPTS/document.sh # Fail out if any setups fail set -e -# Download hosted tool cache AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache +mkdir $AGENT_TOOLSDIRECTORY echo "AGENT_TOOLSDIRECTORY=$AGENT_TOOLSDIRECTORY" | tee -a /etc/environment -azcopy --recursive \ - --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux \ - --destination $AGENT_TOOLSDIRECTORY -# Install tools from hosted tool cache -original_directory=$PWD -setups=$(find $AGENT_TOOLSDIRECTORY -name setup.sh) -for setup in $setups; do - chmod +x $setup; - cd $(dirname $setup); - ./$(basename $setup); - cd $original_directory; +chmod -R 777 $AGENT_TOOLSDIRECTORY + +echo "Installing npm-toolcache..." +TOOLSET_PATH="$INSTALLER_SCRIPT_FOLDER/toolcache.json" + +PACKAGE_LIST=($(jq -r 'keys | .[]' $TOOLSET_PATH)) + +for PACKAGE_NAME in ${PACKAGE_LIST[@]}; do + PACKAGE_VERSIONS=($(jq -r ".[\"$PACKAGE_NAME\"] | .[]" $TOOLSET_PATH)) + for PACKAGE_VERSION in ${PACKAGE_VERSIONS[@]}; do + echo "Install ${PACKAGE_NAME}@${PACKAGE_VERSION}" + npm install ${PACKAGE_NAME}@${PACKAGE_VERSION} --registry=${TOOLCACHE_REGISTRY} + done; done; DocumentInstalledItem "Python:" @@ -33,22 +35,14 @@ for python in $pythons; do DocumentInstalledItemIndent "Python $python" done; -# PyPy is also configured using the setup-python action -pypys=$(ls $AGENT_TOOLSDIRECTORY/PyPy) -for pypy in $pypys; do - DocumentInstalledItemIndent "PyPy $pypy" - # Add symlinks for pypy2 and pypy3 to usr/local/bin, there should only be 2 versions of PyPy in the tools cache that is downloaded - if [ ${pypy:0:1} -eq "3" ] ; then - # add pypy 3.X to PATH - ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy3 /usr/local/bin/pypy3 - else - # add pypy 2.X to PATH - ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy /usr/local/bin/pypy - fi -done; - DocumentInstalledItem "Ruby:" rubys=$(ls $AGENT_TOOLSDIRECTORY/Ruby) for ruby in $rubys; do DocumentInstalledItemIndent "Ruby $ruby" -done; \ No newline at end of file +done; + +DocumentInstalledItem "PyPy:" +pypys=$(ls $AGENT_TOOLSDIRECTORY/PyPy) +for pypy in $pypys; do + DocumentInstalledItemIndent "PyPy $pypy" +done; diff --git a/images/linux/scripts/installers/test-toolcache.sh b/images/linux/scripts/installers/test-toolcache.sh index af2dd359f..2601db4b4 100644 --- a/images/linux/scripts/installers/test-toolcache.sh +++ b/images/linux/scripts/installers/test-toolcache.sh @@ -6,65 +6,69 @@ # Must be procecessed after tool cache setup(hosted-tool-cache.sh). +Test_Hostedtoolcache_Tool() { + TOOL_NAME=$1 + TOOL_EXEC_TEST=$2 + + if [ -d "$AGENT_TOOLSDIRECTORY/$TOOL_NAME" ]; then + cd $AGENT_TOOLSDIRECTORY/$TOOL_NAME + + tool_dirs=($(find . -mindepth 1 -maxdepth 1 -type d | sed "s|^\./||")) + + echo "--------------------------------------------" + echo "$TOOL_NAME versions folders: ${tool_dirs[@]}" + echo "--------------------------------------------" + + if [ -n "$tool_dirs" ]; then + tool_key=$(echo $TOOL_NAME | tr "[:upper:]" "[:lower:]") + package_versions=($(jq -r ".[\"${TOOLCACHE_KEY_VALUE[$tool_key]}\"] | .[]" "$INSTALLER_SCRIPT_FOLDER/toolcache.json")) + + for tool_version in ${package_versions[@]} + do + version_dir=$(find . -name "$tool_version.*" -print -quit) + + echo "Test $AGENT_TOOLSDIRECTORY/$TOOL_NAME/$version_dir:" + + actual_version=$(eval $AGENT_TOOLSDIRECTORY/$TOOL_NAME/$version_dir/$TOOL_EXEC_TEST) + + if [ "$actual_version" = "$tool_version" ]; then + echo "Passed!" + else + echo "Expected: $tool_version; Actual: $actual_version" + + exit 1 + fi + done; + else + echo "$AGENT_TOOLSDIRECTORY/$tool_version does not include any folders" + + exit 1 + fi + else + echo "$AGENT_TOOLSDIRECTORY/$tool_version does not exist" + + exit 1 + fi +} + # Fail out if any tests fail set -e +# define dictionary for key_alias and its values +declare -A TOOLCACHE_KEY_VALUE + +package_list=($(jq -r 'keys | .[]' "$INSTALLER_SCRIPT_FOLDER/toolcache.json")) + +for package_name in ${package_list[@]}; do + # get key alias + key_alias=$(echo $package_name | cut -f2 -d-) + + # set dictionary + TOOLCACHE_KEY_VALUE+=(["$key_alias"]="$package_name") +done; + AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache -# Python test -if [ -d "$AGENT_TOOLSDIRECTORY/Python" ]; then - cd $AGENT_TOOLSDIRECTORY/Python - python_dirs=($(find . -mindepth 1 -maxdepth 1 -type d | sed "s|^\./||")) - echo "Python versions folders: ${python_dirs[@]}" - echo "------------------------------------------" - if [ -n "$python_dirs" ]; then - for version_dir in "${python_dirs[@]}" - do - echo "Test $AGENT_TOOLSDIRECTORY/Python/$version_dir:" - expected_ver=$(echo $version_dir | egrep -o '[0-9]+\.[0-9]+') - actual_ver=$($AGENT_TOOLSDIRECTORY/Python/$version_dir/x64/python -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+') - - if [ "$expected_ver" = "$actual_ver" ]; then - echo "Passed!" - else - echo "Expected: $expected_ver; Actual: $actual_ver" - exit 1 - fi - done - else - echo "$AGENT_TOOLSDIRECTORY/Python does not include any folders" - exit 1 - fi -else - echo "$AGENT_TOOLSDIRECTORY/Python does not exist" - exit 1 -fi - -# Ruby test -if [ -d "$AGENT_TOOLSDIRECTORY/Ruby" ]; then - cd $AGENT_TOOLSDIRECTORY/Ruby - ruby_dirs=($(find . -mindepth 1 -maxdepth 1 -type d | sed "s|^\./||")) - echo "Ruby versions folders: ${ruby_dirs[@]}" - echo "--------------------------------------" - if [ -n "$ruby_dirs" ]; then - for version_dir in "${ruby_dirs[@]}" - do - echo "Test $AGENT_TOOLSDIRECTORY/Ruby/$version_dir:" - expected_ver=$(echo $version_dir | egrep -o '[0-9]+\.[0-9]+') - actual_ver=$($AGENT_TOOLSDIRECTORY/Ruby/$version_dir/x64/bin/ruby -e "puts RUBY_VERSION" | egrep -o '[0-9]+\.[0-9]+') - - if [ "$expected_ver" = "$actual_ver" ]; then - echo "Passed!" - else - echo "Expected: $expected_ver; Actual: $actual_ver" - exit 1 - fi - done - else - echo "$AGENT_TOOLSDIRECTORY/Ruby does not include any folders" - exit 1 - fi -else - echo "$AGENT_TOOLSDIRECTORY/Ruby does not exist" - exit 1 -fi +Test_Hostedtoolcache_Tool "Python" "x64/python -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+'" +Test_Hostedtoolcache_Tool "Ruby" "x64/bin/ruby -e 'puts RUBY_VERSION' | egrep -o '[0-9]+\.[0-9]+'" +Test_Hostedtoolcache_Tool "PyPy" "x64/bin/python -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+' | cut -d '.' -f 1" diff --git a/images/linux/toolcache-1604.json b/images/linux/toolcache-1604.json new file mode 100644 index 000000000..ea7f73c24 --- /dev/null +++ b/images/linux/toolcache-1604.json @@ -0,0 +1,14 @@ +{ + "toolcache-python-ubuntu-1604-x64": [ + "2.7", "3.5", "3.6", "3.7", "3.8" + ], + "toolcache-ruby-ubuntu-1604-x64": [ + "2.4", "2.5", "2.6", "2.7" + ], + "toolcache-pypy-ubuntu-1604-x64": [ + "2", "3" + ], + "toolcache-boost-ubuntu-1604-x64": [ + "1.69" + ] +} \ No newline at end of file diff --git a/images/linux/toolcache-1804.json b/images/linux/toolcache-1804.json new file mode 100644 index 000000000..b076eaf2c --- /dev/null +++ b/images/linux/toolcache-1804.json @@ -0,0 +1,14 @@ +{ + "toolcache-python-ubuntu-1804-x64": [ + "2.7", "3.5", "3.6", "3.7", "3.8" + ], + "toolcache-ruby-ubuntu-1804-x64": [ + "2.4", "2.5", "2.6", "2.7" + ], + "toolcache-pypy-ubuntu-1804-x64": [ + "2", "3" + ], + "toolcache-boost-ubuntu-1804-x64": [ + "1.69" + ] +} \ No newline at end of file diff --git a/images/linux/ubuntu1604.json b/images/linux/ubuntu1604.json index ca7923a39..da60f370a 100644 --- a/images/linux/ubuntu1604.json +++ b/images/linux/ubuntu1604.json @@ -20,7 +20,8 @@ "helper_script_folder": "/imagegeneration/helpers", "vm_size": "Standard_DS2_v2", "capture_name_prefix": "packer", - "image_version": "dev" + "image_version": "dev", + "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/" }, "sensitive-variables": ["client_secret"], "builders": [ @@ -168,19 +169,26 @@ ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, + { + "type": "file", + "source": "{{template_dir}}/toolcache-1604.json", + "destination": "{{user `installer_script_folder`}}/toolcache.json" + }, { "type": "shell", "scripts":[ "{{template_dir}}/scripts/installers/1604/android.sh", "{{template_dir}}/scripts/installers/1604/azpowershell.sh", "{{template_dir}}/scripts/helpers/containercache.sh", - "{{template_dir}}/scripts/installers/1604/hosted-tool-cache.sh", + "{{template_dir}}/scripts/installers/hosted-tool-cache.sh", "{{template_dir}}/scripts/installers/python.sh", "{{template_dir}}/scripts/installers/test-toolcache.sh" ], "environment_vars": [ "METADATA_FILE={{user `metadata_file`}}", - "HELPER_SCRIPTS={{user `helper_script_folder`}}" + "HELPER_SCRIPTS={{user `helper_script_folder`}}", + "TOOLCACHE_REGISTRY={{user `toolcache_registry`}}", + "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}" ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index fa5e66f0b..2803dd911 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -20,7 +20,8 @@ "helper_script_folder": "/imagegeneration/helpers", "vm_size": "Standard_DS2_v2", "capture_name_prefix": "packer", - "image_version": "dev" + "image_version": "dev", + "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/" }, "sensitive-variables": ["client_secret"], "builders": [ @@ -171,19 +172,26 @@ ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, + { + "type": "file", + "source": "{{template_dir}}/toolcache-1804.json", + "destination": "{{user `installer_script_folder`}}/toolcache.json" + }, { "type": "shell", "scripts":[ "{{template_dir}}/scripts/installers/1804/android.sh", "{{template_dir}}/scripts/installers/1804/azpowershell.sh", "{{template_dir}}/scripts/helpers/containercache.sh", - "{{template_dir}}/scripts/installers/1804/hosted-tool-cache.sh", + "{{template_dir}}/scripts/installers/hosted-tool-cache.sh", "{{template_dir}}/scripts/installers/python.sh", "{{template_dir}}/scripts/installers/test-toolcache.sh" ], "environment_vars": [ "METADATA_FILE={{user `metadata_file`}}", - "HELPER_SCRIPTS={{user `helper_script_folder`}}" + "HELPER_SCRIPTS={{user `helper_script_folder`}}", + "TOOLCACHE_REGISTRY={{user `toolcache_registry`}}", + "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}" ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, diff --git a/images/win/Windows2016-Azure.json b/images/win/Windows2016-Azure.json index 493c4df4d..646a12722 100644 --- a/images/win/Windows2016-Azure.json +++ b/images/win/Windows2016-Azure.json @@ -16,6 +16,7 @@ "private_virtual_network_with_public_ip": "{{env `PRIVATE_VIRTUAL_NETWORK_WITH_PUBLIC_IP`}}", "vm_size": "Standard_DS4_v2", + "root_folder": "C:", "image_folder": "C:\\image", "commit_file": "C:\\image\\commit.txt", "metadata_file": "C:\\image\\metadata.txt", @@ -24,7 +25,8 @@ "install_user": "installer", "install_password": null, "capture_name_prefix": "packer", - "image_version": "dev" + "image_version": "dev", + "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/" }, "sensitive-variables": ["install_password", "ssh_password", "client_secret"], "builders": [ @@ -269,6 +271,27 @@ }, { "type": "powershell", + "scripts":[ + "{{ template_dir }}/scripts/Installers/Install-NodeLts.ps1" + ] + }, + { + "type": "powershell", + "scripts":[ + "{{ template_dir }}/scripts/Installers/Install-7zip.ps1" + ] + }, + { + "type": "file", + "source": "{{template_dir}}/toolcache.json", + "destination": "{{user `root_folder`}}/toolcache.json" + }, + { + "type": "powershell", + "environment_vars":[ + "TOOLCACHE_REGISTRY={{ user `toolcache_registry` }}", + "ROOT_FOLDER={{user `root_folder`}}" + ], "scripts":[ "{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1" ] @@ -369,12 +392,6 @@ "{{ template_dir }}/scripts/Installers/Validate-WinAppDriver.ps1" ] }, - { - "type": "powershell", - "scripts":[ - "{{ template_dir }}/scripts/Installers/Install-NodeLts.ps1" - ] - }, { "type": "powershell", "scripts":[ @@ -457,12 +474,6 @@ "{{ template_dir }}/scripts/Installers/Install-AzureCosmosDbEmulator.ps1" ] }, - { - "type": "powershell", - "scripts":[ - "{{ template_dir }}/scripts/Installers/Install-7zip.ps1" - ] - }, { "type": "powershell", "environment_vars": [ @@ -569,6 +580,9 @@ }, { "type": "powershell", + "environment_vars":[ + "ROOT_FOLDER={{user `root_folder`}}" + ], "scripts":[ "{{ template_dir }}/scripts/Installers/Validate-ToolCache.ps1" ] diff --git a/images/win/Windows2019-Azure.json b/images/win/Windows2019-Azure.json index 2424320ae..a31d70879 100644 --- a/images/win/Windows2019-Azure.json +++ b/images/win/Windows2019-Azure.json @@ -16,6 +16,7 @@ "private_virtual_network_with_public_ip": "{{env `PRIVATE_VIRTUAL_NETWORK_WITH_PUBLIC_IP`}}", "vm_size": "Standard_DS4_v2", + "root_folder": "C:", "image_folder": "C:\\image", "commit_file": "C:\\image\\commit.txt", "metadata_file": "C:\\image\\metadata.txt", @@ -24,7 +25,8 @@ "install_user": "installer", "install_password": null, "capture_name_prefix": "packer", - "image_version": "dev" + "image_version": "dev", + "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/" }, "sensitive-variables": ["install_password", "ssh_password", "client_secret"], "builders": [ @@ -238,6 +240,27 @@ }, { "type": "powershell", + "scripts":[ + "{{ template_dir }}/scripts/Installers/Install-NodeLts.ps1" + ] + }, + { + "type": "powershell", + "scripts":[ + "{{ template_dir }}/scripts/Installers/Install-7zip.ps1" + ] + }, + { + "type": "file", + "source": "{{template_dir}}/toolcache.json", + "destination": "{{user `root_folder`}}/toolcache.json" + }, + { + "type": "powershell", + "environment_vars":[ + "TOOLCACHE_REGISTRY={{ user `toolcache_registry` }}", + "ROOT_FOLDER={{user `root_folder`}}" + ], "scripts":[ "{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1" ] @@ -338,12 +361,6 @@ "{{ template_dir }}/scripts/Installers/Validate-WinAppDriver.ps1" ] }, - { - "type": "powershell", - "scripts":[ - "{{ template_dir }}/scripts/Installers/Install-NodeLts.ps1" - ] - }, { "type": "powershell", "scripts":[ @@ -426,12 +443,6 @@ "{{ template_dir }}/scripts/Installers/Install-AzureCosmosDbEmulator.ps1" ] }, - { - "type": "powershell", - "scripts":[ - "{{ template_dir }}/scripts/Installers/Install-7zip.ps1" - ] - }, { "type": "powershell", "environment_vars": [ @@ -538,6 +549,9 @@ }, { "type": "powershell", + "environment_vars":[ + "ROOT_FOLDER={{user `root_folder`}}" + ], "scripts":[ "{{ template_dir }}/scripts/Installers/Validate-ToolCache.ps1" ] diff --git a/images/win/scripts/Installers/Download-ToolCache.ps1 b/images/win/scripts/Installers/Download-ToolCache.ps1 index b7a49a285..b031a827e 100644 --- a/images/win/scripts/Installers/Download-ToolCache.ps1 +++ b/images/win/scripts/Installers/Download-ToolCache.ps1 @@ -1,51 +1,48 @@ ################################################################################ ## File: Download-ToolCache.ps1 +## Team: CI-Build ## Desc: Download tool cache ################################################################################ -Function InstallTool -{ - Param - ( - [System.Object]$ExecutablePath +Function Install-NpmPackage { + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [System.String] + $Name, + [Parameter(Mandatory=$true)] + [System.String] + $NpmRegistry ) - Write-Host $ExecutablePath.DirectoryName - Set-Location -Path $ExecutablePath.DirectoryName - Get-Location | Write-Host - if (Test-Path 'tool.zip') - { - Expand-Archive 'tool.zip' -DestinationPath '.' - } - cmd.exe /c 'install_to_tools_cache.bat' + Write-Host "Installing npm '$Name' package from '$NpmRegistry'" + + npm install $Name --registry=$NpmRegistry } -$SourceUrl = "https://vstsagenttools.blob.core.windows.net/tools" - +# HostedToolCache Path $Dest = "C:/" - $Path = "hostedtoolcache/windows" - -$env:Path = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy;" + $env:Path - -Write-Host "Started AzCopy from $SourceUrl to $Dest" - -AzCopy /Source:$SourceUrl /Dest:$Dest /S /V /Pattern:$Path - $ToolsDirectory = $Dest + $Path -$current = Get-Location -Set-Location -Path $ToolsDirectory - -Get-ChildItem -Recurse -Depth 4 -Filter install_to_tools_cache.bat | ForEach-Object { - #In order to work correctly Python 3.4 x86 must be installed after x64, this is achieved by current toolcache catalog structure - InstallTool($_) -} - -Set-Location -Path $current - +# Define AGENT_TOOLSDIRECTORY environment variable +$env:AGENT_TOOLSDIRECTORY = $ToolsDirectory setx AGENT_TOOLSDIRECTORY $ToolsDirectory /M +# Install HostedToolCache tools via NPM +$ToolVersionsFileContent = Get-Content -Path "$env:ROOT_FOLDER/toolcache.json" -Raw +$ToolVersions = ConvertFrom-Json -InputObject $ToolVersionsFileContent + +$ToolVersions.PSObject.Properties | ForEach-Object { + $PackageName = $_.Name + $PackageVersions = $_.Value + $NpmPackages = $PackageVersions | ForEach-Object { "$PackageName@$_" } + foreach($NpmPackage in $NpmPackages) { + Install-NpmPackage -Name $NpmPackage -NpmRegistry $env:TOOLCACHE_REGISTRY + } +} + #junction point from the previous Python2 directory to the toolcache Python2 +Write-Host "Create symlink to Python2" $python2Dir = (Get-Item -Path ($ToolsDirectory + '/Python/2.7*/x64')).FullName cmd.exe /c mklink /d "C:\Python27amd64" "$python2Dir" diff --git a/images/win/scripts/Installers/Install-Boost.ps1 b/images/win/scripts/Installers/Install-Boost.ps1 index a076a47af..281363336 100644 --- a/images/win/scripts/Installers/Install-Boost.ps1 +++ b/images/win/scripts/Installers/Install-Boost.ps1 @@ -1,28 +1,16 @@ ################################################################################ -## File: Install-Boost.ps1 +## File: Set-BoostRoot.ps1 +## Team: CI-Build ## Desc: Install boost using tool cache ################################################################################ -$ToolCache = 'C:\hostedtoolcache\windows\boost' $BoostDirectory = Join-Path -Path $env:ProgramFiles -ChildPath "Boost" -$BoostVersionsToInstall = $env:BOOST_VERSIONS.split(',') +$BoostVersions = $env:BOOST_VERSIONS.split(',') $BoostDefault = $env:BOOST_DEFAULT -foreach($BoostVersion in $BoostVersionsToInstall) +foreach($BoostVersion in $BoostVersions) { - $ZipName = Join-Path -Path $ToolCache -ChildPath "boost_${BoostVersion}_msvc-14.1.zip" - - if (-Not (Test-Path $ZipName)) - { - Write-Host "$ZipName not found" - exit 1 - } - - Write-Host "Expanding $ZipName" - $BoostInstallationDir = Join-Path -Path $BoostDirectory -ChildPath $BoostVersion - # Expand-Archive slower for 70% than 7z - & "$env:ProgramFiles\7-Zip\7z.exe" x $ZipName -o"$BoostDirectory" -y $EnvBoostPath = "BOOST_ROOT_{0}" -f ($BoostVersion.Replace('.', '_')) setx $EnvBoostPath $BoostInstallationDir /M | Out-Null @@ -36,10 +24,3 @@ foreach($BoostVersion in $BoostVersionsToInstall) setx BOOST_ROOT $BoostInstallationDir /M | Out-Null } } - -# Removing boost cache folder -Write-Host "Removing boost cache folder" -if (Test-Path $ToolCache) -{ - Remove-Item -Path $ToolCache -Force -Recurse -} diff --git a/images/win/scripts/Installers/Validate-ToolCache.ps1 b/images/win/scripts/Installers/Validate-ToolCache.ps1 index 832cf13c3..cde063511 100644 --- a/images/win/scripts/Installers/Validate-ToolCache.ps1 +++ b/images/win/scripts/Installers/Validate-ToolCache.ps1 @@ -12,6 +12,21 @@ function GetChildFolders { return Get-ChildItem -Path $Path -Directory -Name } +function Get-ToolcachePackages { + $toolcachePath = Join-Path $env:ROOT_FOLDER "toolcache.json" + return Get-Content -Raw $toolcachePath | ConvertFrom-Json +} + +$packages = (Get-ToolcachePackages).PSObject.Properties | ForEach-Object { + $packageNameParts = $_.Name.Split("-") + $toolName = $packageNameParts[1] + return [PSCustomObject] @{ + ToolName = $packageNameParts[1] + Versions = $_.Value + Arch = $packageNameParts[3] + } +} + function ToolcacheTest { param ( [Parameter(Mandatory = $True)] @@ -22,36 +37,53 @@ function ToolcacheTest { if (Test-Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName") { $description = "" - [array]$versions = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName" - if ($versions.count -gt 0){ - foreach ($version in $versions) + [array]$installedVersions = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName" + if ($installedVersions.count -gt 0){ + $softwarePackages = $packages | Where-Object { $_.ToolName -eq $SoftwareName } + foreach($softwarePackage in $softwarePackages) { - $architectures = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$version" - - Write-Host "$SoftwareName version - $version : $([system.String]::Join(",", $architectures))" - - foreach ($arch in $architectures) + foreach ($version in $softwarePackage.Versions) { - $path = "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$version\$arch" - foreach ($test in $ExecTests) - { - if (Test-Path "$path\$test") - { - Write-Host "$SoftwareName($test) $version($arch) is successfully installed:" - Write-Host (& "$path\$test" --version) + $foundVersion = $installedVersions | where { $_.StartsWith($version) } + + if ($foundVersion -ne $null){ + + $architectures = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$foundVersion" + + $softwareArch = $softwarePackage.Arch + + if ($architectures -Contains $softwareArch) { + $path = "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$foundVersion\$softwareArch" + foreach ($test in $ExecTests) + { + if (Test-Path "$path\$test") + { + Write-Host "$SoftwareName($test) $foundVersion($softwareArch) is successfully installed:" + Write-Host (& "$path\$test" --version) + } + else + { + Write-Host "$SoftwareName($test) $foundVersion ($softwareArch) is not installed" + exit 1 + } + } + $description += "_Version:_ $foundVersion ($softwareArch)
" } else { - Write-Host "$SoftwareName($test) $version ($arch) is not installed" + Write-Host "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$foundVersion does not include required architecture" exit 1 } } - - $description += "_Version:_ $version ($arch)
" + else + { + Write-Host "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$version.* was not found" + exit 1 + } } - } - Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $description + Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $description + } } else { diff --git a/images/win/toolcache.json b/images/win/toolcache.json new file mode 100644 index 000000000..32aae3d50 --- /dev/null +++ b/images/win/toolcache.json @@ -0,0 +1,17 @@ +{ + "toolcache-python-windows-x64": [ + "2.7", "3.5", "3.6", "3.7", "3.8" + ], + "toolcache-python-windows-x86": [ + "2.7", "3.5", "3.6", "3.7", "3.8" + ], + "toolcache-ruby-windows-x64": [ + "2.4", "2.5", "2.6", "2.7" + ], + "toolcache-pypy-windows-x86": [ + "2", "3" + ], + "toolcache-boost-windows-x64": [ + "1.69" + ] +} \ No newline at end of file