diff --git a/images/linux/scripts/installers/1804/hosted-tool-cache.sh b/images/linux/scripts/installers/1804/hosted-tool-cache.sh deleted file mode 100644 index 10180a240..000000000 --- a/images/linux/scripts/installers/1804/hosted-tool-cache.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -################################################################################ -## File: hosted-tool-cache.sh -## Desc: Downloads and installs hosted tools cache -################################################################################ - -# Source the helpers for use with the script -source $HELPER_SCRIPTS/document.sh - -# Fail out if any setups fail -set -e - -# Download hosted tool cache -AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache -echo "AGENT_TOOLSDIRECTORY=$AGENT_TOOLSDIRECTORY" | tee -a /etc/environment -azcopy --recursive \ - --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/ubuntu-1804 \ - --destination $AGENT_TOOLSDIRECTORY - -# Install tools from hosted tool cache -original_directory=$PWD -setups=$(find $AGENT_TOOLSDIRECTORY -name setup.sh) -for setup in $setups; do - chmod +x $setup; - cd $(dirname $setup); - ./$(basename $setup); - cd $original_directory; -done; - -DocumentInstalledItem "Python:" -pythons=$(ls $AGENT_TOOLSDIRECTORY/Python) -for python in $pythons; do - DocumentInstalledItemIndent "Python $python" -done; - -# PyPy is also configured using the setup-python action -pypys=$(ls $AGENT_TOOLSDIRECTORY/PyPy) -for pypy in $pypys; do - DocumentInstalledItemIndent "PyPy $pypy" - # Add symlinks for pypy2 and pypy3 to usr/local/bin, there should only be 2 versions of PyPy in the tools cache that is downloaded - if [ ${pypy:0:1} -eq "3" ] ; then - # add pypy 3.X to PATH - ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy3 /usr/local/bin/pypy3 - else - # add pypy 2.X to PATH - ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy /usr/local/bin/pypy - fi -done; - -DocumentInstalledItem "Ruby:" -rubys=$(ls $AGENT_TOOLSDIRECTORY/Ruby) -for ruby in $rubys; do - DocumentInstalledItemIndent "Ruby $ruby" -done; \ No newline at end of file diff --git a/images/linux/scripts/installers/boost.sh b/images/linux/scripts/installers/boost.sh index 3b7d6b043..673b1f61e 100644 --- a/images/linux/scripts/installers/boost.sh +++ b/images/linux/scripts/installers/boost.sh @@ -7,7 +7,6 @@ # Source the helpers for use with the script source $HELPER_SCRIPTS/document.sh -BOOST_ZIP_PATH=/opt/hostedtoolcache/Boost BOOST_LIB=/usr/local/share/boost # Install Boost @@ -15,9 +14,6 @@ for BOOST_VERSION in ${BOOST_VERSIONS//,/ } do BOOST_SYMLINK_VER=`echo "${BOOST_VERSION//[.]/_}"` BOOST_ROOT="BOOST_ROOT_$BOOST_SYMLINK_VER" - BOOST_ZIP="boost_`echo $BOOST_VERSION`_gcc.zip" - - unzip $BOOST_ZIP_PATH/$BOOST_ZIP -d $BOOST_LIB echo "$BOOST_ROOT=$BOOST_LIB/$BOOST_VERSION" | tee -a /etc/environment if [[ $BOOST_VERSION == $BOOST_DEFAULT ]]; then @@ -26,6 +22,3 @@ do DocumentInstalledItem "Boost C++ Libraries $BOOST_VERSION" done - -# Deleting archives with Boost Libraries -rm -rf $BOOST_ZIP_PATH diff --git a/images/linux/scripts/installers/1604/hosted-tool-cache.sh b/images/linux/scripts/installers/hosted-tool-cache.sh similarity index 51% rename from images/linux/scripts/installers/1604/hosted-tool-cache.sh rename to images/linux/scripts/installers/hosted-tool-cache.sh index c68f9747e..66ab532ea 100644 --- a/images/linux/scripts/installers/1604/hosted-tool-cache.sh +++ b/images/linux/scripts/installers/hosted-tool-cache.sh @@ -10,21 +10,23 @@ source $HELPER_SCRIPTS/document.sh # Fail out if any setups fail set -e -# Download hosted tool cache AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache +mkdir $AGENT_TOOLSDIRECTORY echo "AGENT_TOOLSDIRECTORY=$AGENT_TOOLSDIRECTORY" | tee -a /etc/environment -azcopy --recursive \ - --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux \ - --destination $AGENT_TOOLSDIRECTORY -# Install tools from hosted tool cache -original_directory=$PWD -setups=$(find $AGENT_TOOLSDIRECTORY -name setup.sh) -for setup in $setups; do - chmod +x $setup; - cd $(dirname $setup); - ./$(basename $setup); - cd $original_directory; +chmod -R 777 $AGENT_TOOLSDIRECTORY + +echo "Installing npm-toolcache..." +TOOLSET_PATH="$INSTALLER_SCRIPT_FOLDER/toolcache.json" + +PACKAGE_LIST=($(jq -r 'keys | .[]' $TOOLSET_PATH)) + +for PACKAGE_NAME in ${PACKAGE_LIST[@]}; do + PACKAGE_VERSIONS=($(jq -r ".[\"$PACKAGE_NAME\"] | .[]" $TOOLSET_PATH)) + for PACKAGE_VERSION in ${PACKAGE_VERSIONS[@]}; do + echo "Install ${PACKAGE_NAME}@${PACKAGE_VERSION}" + npm install ${PACKAGE_NAME}@${PACKAGE_VERSION} --registry=${TOOLCACHE_REGISTRY} + done; done; DocumentInstalledItem "Python:" @@ -33,22 +35,14 @@ for python in $pythons; do DocumentInstalledItemIndent "Python $python" done; -# PyPy is also configured using the setup-python action -pypys=$(ls $AGENT_TOOLSDIRECTORY/PyPy) -for pypy in $pypys; do - DocumentInstalledItemIndent "PyPy $pypy" - # Add symlinks for pypy2 and pypy3 to usr/local/bin, there should only be 2 versions of PyPy in the tools cache that is downloaded - if [ ${pypy:0:1} -eq "3" ] ; then - # add pypy 3.X to PATH - ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy3 /usr/local/bin/pypy3 - else - # add pypy 2.X to PATH - ln -s $AGENT_TOOLSDIRECTORY/PyPy/$pypy/x64/bin/pypy /usr/local/bin/pypy - fi -done; - DocumentInstalledItem "Ruby:" rubys=$(ls $AGENT_TOOLSDIRECTORY/Ruby) for ruby in $rubys; do DocumentInstalledItemIndent "Ruby $ruby" -done; \ No newline at end of file +done; + +DocumentInstalledItem "PyPy:" +pypys=$(ls $AGENT_TOOLSDIRECTORY/PyPy) +for pypy in $pypys; do + DocumentInstalledItemIndent "PyPy $pypy" +done; diff --git a/images/linux/scripts/installers/test-toolcache.sh b/images/linux/scripts/installers/test-toolcache.sh index af2dd359f..785218b3e 100644 --- a/images/linux/scripts/installers/test-toolcache.sh +++ b/images/linux/scripts/installers/test-toolcache.sh @@ -9,6 +9,22 @@ # Fail out if any tests fail set -e +# defune array of key aliases +key_alias_array=() +bash --version +# define dicionary for key_alias and its values +declare -A toolcache_key_value +PACKAGE_LIST=($(jq -r 'keys | .[]' "$INSTALLER_SCRIPT_FOLDER/toolcache.json")) +for PACKAGE_NAME in ${PACKAGE_LIST[@]}; do + + # get key alias + key_alias=$(echo $PACKAGE_NAME | cut -f2 -d-) + echo $key_alias + + # set dictionary + toolcache_key_value+=(["$key_alias"]="$PACKAGE_NAME") +done; + AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache # Python test @@ -18,19 +34,19 @@ if [ -d "$AGENT_TOOLSDIRECTORY/Python" ]; then echo "Python versions folders: ${python_dirs[@]}" echo "------------------------------------------" if [ -n "$python_dirs" ]; then - for version_dir in "${python_dirs[@]}" + PACKAGE_VERSIONS=($(jq -r ".[\"${toolcache_key_value[python]}\"] | .[]" "$INSTALLER_SCRIPT_FOLDER/toolcache.json")) + for python_version in ${PACKAGE_VERSIONS[@]} do + version_dir=$(find . -name "$python_version.*" -print -quit) echo "Test $AGENT_TOOLSDIRECTORY/Python/$version_dir:" - expected_ver=$(echo $version_dir | egrep -o '[0-9]+\.[0-9]+') actual_ver=$($AGENT_TOOLSDIRECTORY/Python/$version_dir/x64/python -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+') - - if [ "$expected_ver" = "$actual_ver" ]; then + if [ "$actual_ver" = "$python_version" ]; then echo "Passed!" else - echo "Expected: $expected_ver; Actual: $actual_ver" + echo "Expected: $python_version; Actual: $actual_ver" exit 1 fi - done + done; else echo "$AGENT_TOOLSDIRECTORY/Python does not include any folders" exit 1 @@ -47,19 +63,19 @@ if [ -d "$AGENT_TOOLSDIRECTORY/Ruby" ]; then echo "Ruby versions folders: ${ruby_dirs[@]}" echo "--------------------------------------" if [ -n "$ruby_dirs" ]; then - for version_dir in "${ruby_dirs[@]}" + PACKAGE_VERSIONS=($(jq -r ".[\"${toolcache_key_value[ruby]}\"] | .[]" "$INSTALLER_SCRIPT_FOLDER/toolcache.json")) + for ruby_version in ${PACKAGE_VERSIONS[@]} do + version_dir=$(find . -name "$ruby_version.*" -print -quit) echo "Test $AGENT_TOOLSDIRECTORY/Ruby/$version_dir:" - expected_ver=$(echo $version_dir | egrep -o '[0-9]+\.[0-9]+') actual_ver=$($AGENT_TOOLSDIRECTORY/Ruby/$version_dir/x64/bin/ruby -e "puts RUBY_VERSION" | egrep -o '[0-9]+\.[0-9]+') - - if [ "$expected_ver" = "$actual_ver" ]; then - echo "Passed!" + if [ "$actual_ver" = "$ruby_version" ]; then + echo "Passed!" else - echo "Expected: $expected_ver; Actual: $actual_ver" - exit 1 + echo "Expected: $ruby_version; Actual: $actual_ver" + exit 1 fi - done + done; else echo "$AGENT_TOOLSDIRECTORY/Ruby does not include any folders" exit 1 @@ -68,3 +84,34 @@ else echo "$AGENT_TOOLSDIRECTORY/Ruby does not exist" exit 1 fi + +# PyPy tests +if [ -d "$AGENT_TOOLSDIRECTORY/PyPy" ]; then + cd $AGENT_TOOLSDIRECTORY/PyPy + pypy_dirs=($(find . -mindepth 1 -maxdepth 1 -type d | sed "s|^\./||")) + echo "PyPy versions folders: ${pypy_dirs[@]}" + echo "------------------------------------------" + if [ -n "$pypy_dirs" ]; then + PACKAGE_VERSIONS=($(jq -r ".[\"${toolcache_key_value[pypy]}\"] | .[]" "$INSTALLER_SCRIPT_FOLDER/toolcache.json")) + for pypy_version in ${PACKAGE_VERSIONS[@]} + do + version_dir=$(find . -name "$pypy_version.*" -print -quit) + pypy_path=$(find . -regex "$version_dir/x64/bin/pypy[0-9]*" -print -quit) + echo $pypy_path + actual_ver=$($pypy_path -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+' | cut -d '.' -f 1) + echo "actual_ver = $actual_ver : pypy_version = $pypy_version" + if [ "$actual_ver" = "$pypy_version" ]; then + echo "Passed!" + else + echo "Expected: $pypy_version; Actual: $actual_ver" + exit 1 + fi + done; + else + echo "$AGENT_TOOLSDIRECTORY/PyPy does not include any folders" + exit 1 + fi +else + echo "$AGENT_TOOLSDIRECTORY/PyPy does not exist" + exit 1 +fi \ No newline at end of file diff --git a/images/linux/toolcache-1604.json b/images/linux/toolcache-1604.json new file mode 100644 index 000000000..2674d57af --- /dev/null +++ b/images/linux/toolcache-1604.json @@ -0,0 +1,14 @@ +{ + "toolcache-python-ubuntu-1604-x64": [ + "2.7", "3.5", "3.6", "3.7", "3.8" + ], + "toolcache-ruby-ubuntu-1604-x64": [ + "2.4", "2.5", "2.6" + ], + "toolcache-pypy-ubuntu-1604-x64": [ + "2", "3" + ], + "toolcache-boost-ubuntu-1604-x64": [ + "1.69" + ] +} \ No newline at end of file diff --git a/images/linux/toolcache-1804.json b/images/linux/toolcache-1804.json new file mode 100644 index 000000000..ca75563cd --- /dev/null +++ b/images/linux/toolcache-1804.json @@ -0,0 +1,14 @@ +{ + "toolcache-python-ubuntu-1804-x64": [ + "2.7", "3.5", "3.6", "3.7", "3.8" + ], + "toolcache-ruby-ubuntu-1804-x64": [ + "2.4", "2.5", "2.6" + ], + "toolcache-pypy-ubuntu-1804-x64": [ + "2", "3" + ], + "toolcache-boost-ubuntu-1804-x64": [ + "1.69" + ] +} \ No newline at end of file diff --git a/images/linux/ubuntu1604.json b/images/linux/ubuntu1604.json index a1e9d27fa..650f3daec 100644 --- a/images/linux/ubuntu1604.json +++ b/images/linux/ubuntu1604.json @@ -20,7 +20,8 @@ "helper_script_folder": "/imagegeneration/helpers", "vm_size": "Standard_DS2_v2", "capture_name_prefix": "packer", - "image_version": "dev" + "image_version": "dev", + "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/" }, "builders": [ { @@ -167,19 +168,26 @@ ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, + { + "type": "file", + "source": "{{template_dir}}/toolcache-1604.json", + "destination": "{{user `installer_script_folder`}}/toolcache.json" + }, { "type": "shell", "scripts":[ "{{template_dir}}/scripts/installers/1604/android.sh", "{{template_dir}}/scripts/installers/1604/azpowershell.sh", "{{template_dir}}/scripts/helpers/containercache.sh", - "{{template_dir}}/scripts/installers/1604/hosted-tool-cache.sh", + "{{template_dir}}/scripts/installers/hosted-tool-cache.sh", "{{template_dir}}/scripts/installers/python.sh", "{{template_dir}}/scripts/installers/test-toolcache.sh" ], "environment_vars": [ "METADATA_FILE={{user `metadata_file`}}", - "HELPER_SCRIPTS={{user `helper_script_folder`}}" + "HELPER_SCRIPTS={{user `helper_script_folder`}}", + "TOOLCACHE_REGISTRY={{user `toolcache_registry`}}", + "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}" ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index 2c10a6822..5c59b4833 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -20,7 +20,8 @@ "helper_script_folder": "/imagegeneration/helpers", "vm_size": "Standard_DS2_v2", "capture_name_prefix": "packer", - "image_version": "dev" + "image_version": "dev", + "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/" }, "builders": [ { @@ -170,19 +171,26 @@ ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, + { + "type": "file", + "source": "{{template_dir}}/toolcache-1804.json", + "destination": "{{user `installer_script_folder`}}/toolcache.json" + }, { "type": "shell", "scripts":[ "{{template_dir}}/scripts/installers/1804/android.sh", "{{template_dir}}/scripts/installers/1804/azpowershell.sh", "{{template_dir}}/scripts/helpers/containercache.sh", - "{{template_dir}}/scripts/installers/1804/hosted-tool-cache.sh", + "{{template_dir}}/scripts/installers/hosted-tool-cache.sh", "{{template_dir}}/scripts/installers/python.sh", "{{template_dir}}/scripts/installers/test-toolcache.sh" ], "environment_vars": [ "METADATA_FILE={{user `metadata_file`}}", - "HELPER_SCRIPTS={{user `helper_script_folder`}}" + "HELPER_SCRIPTS={{user `helper_script_folder`}}", + "TOOLCACHE_REGISTRY={{user `toolcache_registry`}}", + "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}" ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, diff --git a/images/win/Windows2016-Azure.json b/images/win/Windows2016-Azure.json index c51dcdfe5..8c93fad94 100644 --- a/images/win/Windows2016-Azure.json +++ b/images/win/Windows2016-Azure.json @@ -16,6 +16,7 @@ "private_virtual_network_with_public_ip": "{{env `PRIVATE_VIRTUAL_NETWORK_WITH_PUBLIC_IP`}}", "vm_size": "Standard_DS4_v2", + "root_folder": "C:", "image_folder": "C:\\image", "commit_file": "C:\\image\\commit.txt", "metadata_file": "C:\\image\\metadata.txt", @@ -24,7 +25,8 @@ "install_user": "installer", "install_password": null, "capture_name_prefix": "packer", - "image_version": "dev" + "image_version": "dev", + "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/" }, "builders": [ { @@ -268,6 +270,27 @@ }, { "type": "powershell", + "scripts":[ + "{{ template_dir }}/scripts/Installers/Install-NodeLts.ps1" + ] + }, + { + "type": "powershell", + "scripts":[ + "{{ template_dir }}/scripts/Installers/Install-7zip.ps1" + ] + }, + { + "type": "file", + "source": "{{template_dir}}/toolcache.json", + "destination": "{{user `root_folder`}}/toolcache.json" + }, + { + "type": "powershell", + "environment_vars":[ + "TOOLCACHE_REGISTRY={{ user `toolcache_registry` }}", + "ROOT_FOLDER={{user `root_folder`}}" + ], "scripts":[ "{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1" ] @@ -362,12 +385,6 @@ "{{ template_dir }}/scripts/Installers/Validate-WinAppDriver.ps1" ] }, - { - "type": "powershell", - "scripts":[ - "{{ template_dir }}/scripts/Installers/Install-NodeLts.ps1" - ] - }, { "type": "powershell", "scripts":[ @@ -450,12 +467,6 @@ "{{ template_dir }}/scripts/Installers/Install-AzureCosmosDbEmulator.ps1" ] }, - { - "type": "powershell", - "scripts":[ - "{{ template_dir }}/scripts/Installers/Install-7zip.ps1" - ] - }, { "type": "powershell", "environment_vars": [ @@ -562,6 +573,9 @@ }, { "type": "powershell", + "environment_vars":[ + "ROOT_FOLDER={{user `root_folder`}}" + ], "scripts":[ "{{ template_dir }}/scripts/Installers/Validate-ToolCache.ps1" ] diff --git a/images/win/Windows2019-Azure.json b/images/win/Windows2019-Azure.json index 6875b7f07..5251a965b 100644 --- a/images/win/Windows2019-Azure.json +++ b/images/win/Windows2019-Azure.json @@ -16,6 +16,7 @@ "private_virtual_network_with_public_ip": "{{env `PRIVATE_VIRTUAL_NETWORK_WITH_PUBLIC_IP`}}", "vm_size": "Standard_DS4_v2", + "root_folder": "C:", "image_folder": "C:\\image", "commit_file": "C:\\image\\commit.txt", "metadata_file": "C:\\image\\metadata.txt", @@ -24,7 +25,8 @@ "install_user": "installer", "install_password": null, "capture_name_prefix": "packer", - "image_version": "dev" + "image_version": "dev", + "toolcache_registry": "https://buildcanary.pkgs.visualstudio.com/PipelineCanary/_packaging/hostedtoolcache/npm/registry/" }, "sensitive-variables": ["install_password", "ssh_password", "client_secret"], "builders": [ @@ -238,6 +240,27 @@ }, { "type": "powershell", + "scripts":[ + "{{ template_dir }}/scripts/Installers/Install-NodeLts.ps1" + ] + }, + { + "type": "powershell", + "scripts":[ + "{{ template_dir }}/scripts/Installers/Install-7zip.ps1" + ] + }, + { + "type": "file", + "source": "{{template_dir}}/toolcache.json", + "destination": "{{user `root_folder`}}/toolcache.json" + }, + { + "type": "powershell", + "environment_vars":[ + "TOOLCACHE_REGISTRY={{ user `toolcache_registry` }}", + "ROOT_FOLDER={{user `root_folder`}}" + ], "scripts":[ "{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1" ] @@ -332,12 +355,6 @@ "{{ template_dir }}/scripts/Installers/Validate-WinAppDriver.ps1" ] }, - { - "type": "powershell", - "scripts":[ - "{{ template_dir }}/scripts/Installers/Install-NodeLts.ps1" - ] - }, { "type": "powershell", "scripts":[ @@ -420,12 +437,6 @@ "{{ template_dir }}/scripts/Installers/Install-AzureCosmosDbEmulator.ps1" ] }, - { - "type": "powershell", - "scripts":[ - "{{ template_dir }}/scripts/Installers/Install-7zip.ps1" - ] - }, { "type": "powershell", "environment_vars": [ @@ -532,6 +543,9 @@ }, { "type": "powershell", + "environment_vars":[ + "ROOT_FOLDER={{user `root_folder`}}" + ], "scripts":[ "{{ template_dir }}/scripts/Installers/Validate-ToolCache.ps1" ] diff --git a/images/win/scripts/Installers/Download-ToolCache.ps1 b/images/win/scripts/Installers/Download-ToolCache.ps1 index b7a49a285..967046d09 100644 --- a/images/win/scripts/Installers/Download-ToolCache.ps1 +++ b/images/win/scripts/Installers/Download-ToolCache.ps1 @@ -1,51 +1,62 @@ ################################################################################ ## File: Download-ToolCache.ps1 +## Team: CI-Build ## Desc: Download tool cache ################################################################################ -Function InstallTool -{ - Param - ( - [System.Object]$ExecutablePath +Function Install-NpmPackage { + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [System.String] + $Name, + [Parameter(Mandatory=$true)] + [System.String] + $NpmRegistry ) - Write-Host $ExecutablePath.DirectoryName - Set-Location -Path $ExecutablePath.DirectoryName - Get-Location | Write-Host - if (Test-Path 'tool.zip') - { + Write-Host "Installing npm '$Name' package from '$NpmRegistry'" + + npm install $Name --registry=$NpmRegistry +} + +Function InstallTool { + [CmdletBinding()] + param( + [System.IO.FileInfo]$ExecutablePath + ) + + Set-Location -Path $ExecutablePath.DirectoryName -PassThru | Write-Host + if (Test-Path 'tool.zip') { Expand-Archive 'tool.zip' -DestinationPath '.' } cmd.exe /c 'install_to_tools_cache.bat' } -$SourceUrl = "https://vstsagenttools.blob.core.windows.net/tools" - +# HostedToolCache Path $Dest = "C:/" - $Path = "hostedtoolcache/windows" - -$env:Path = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy;" + $env:Path - -Write-Host "Started AzCopy from $SourceUrl to $Dest" - -AzCopy /Source:$SourceUrl /Dest:$Dest /S /V /Pattern:$Path - $ToolsDirectory = $Dest + $Path -$current = Get-Location -Set-Location -Path $ToolsDirectory - -Get-ChildItem -Recurse -Depth 4 -Filter install_to_tools_cache.bat | ForEach-Object { - #In order to work correctly Python 3.4 x86 must be installed after x64, this is achieved by current toolcache catalog structure - InstallTool($_) -} - -Set-Location -Path $current - +# Define AGENT_TOOLSDIRECTORY environment variable +$env:AGENT_TOOLSDIRECTORY = $ToolsDirectory setx AGENT_TOOLSDIRECTORY $ToolsDirectory /M +# Install tools form NPM + +$ToolVersionsFileContent = Get-Content -Path "$env:ROOT_FOLDER/toolcache.json" -Raw +$ToolVersions = ConvertFrom-Json -InputObject $ToolVersionsFileContent + +$ToolVersions.PSObject.Properties | ForEach-Object { + $PackageName = $_.Name + $PackageVersions = $_.Value + $NpmPackages = $PackageVersions | ForEach-Object { "$PackageName@$_" } + foreach($NpmPackage in $NpmPackages) { + Install-NpmPackage -Name $NpmPackage -NpmRegistry $env:TOOLCACHE_REGISTRY + } +} + #junction point from the previous Python2 directory to the toolcache Python2 +Write-Host "Create symlink to Python2" $python2Dir = (Get-Item -Path ($ToolsDirectory + '/Python/2.7*/x64')).FullName cmd.exe /c mklink /d "C:\Python27amd64" "$python2Dir" diff --git a/images/win/scripts/Installers/Install-Boost.ps1 b/images/win/scripts/Installers/Install-Boost.ps1 index a076a47af..281363336 100644 --- a/images/win/scripts/Installers/Install-Boost.ps1 +++ b/images/win/scripts/Installers/Install-Boost.ps1 @@ -1,28 +1,16 @@ ################################################################################ -## File: Install-Boost.ps1 +## File: Set-BoostRoot.ps1 +## Team: CI-Build ## Desc: Install boost using tool cache ################################################################################ -$ToolCache = 'C:\hostedtoolcache\windows\boost' $BoostDirectory = Join-Path -Path $env:ProgramFiles -ChildPath "Boost" -$BoostVersionsToInstall = $env:BOOST_VERSIONS.split(',') +$BoostVersions = $env:BOOST_VERSIONS.split(',') $BoostDefault = $env:BOOST_DEFAULT -foreach($BoostVersion in $BoostVersionsToInstall) +foreach($BoostVersion in $BoostVersions) { - $ZipName = Join-Path -Path $ToolCache -ChildPath "boost_${BoostVersion}_msvc-14.1.zip" - - if (-Not (Test-Path $ZipName)) - { - Write-Host "$ZipName not found" - exit 1 - } - - Write-Host "Expanding $ZipName" - $BoostInstallationDir = Join-Path -Path $BoostDirectory -ChildPath $BoostVersion - # Expand-Archive slower for 70% than 7z - & "$env:ProgramFiles\7-Zip\7z.exe" x $ZipName -o"$BoostDirectory" -y $EnvBoostPath = "BOOST_ROOT_{0}" -f ($BoostVersion.Replace('.', '_')) setx $EnvBoostPath $BoostInstallationDir /M | Out-Null @@ -36,10 +24,3 @@ foreach($BoostVersion in $BoostVersionsToInstall) setx BOOST_ROOT $BoostInstallationDir /M | Out-Null } } - -# Removing boost cache folder -Write-Host "Removing boost cache folder" -if (Test-Path $ToolCache) -{ - Remove-Item -Path $ToolCache -Force -Recurse -} diff --git a/images/win/scripts/Installers/Validate-ToolCache.ps1 b/images/win/scripts/Installers/Validate-ToolCache.ps1 index 832cf13c3..df61f6f6c 100644 --- a/images/win/scripts/Installers/Validate-ToolCache.ps1 +++ b/images/win/scripts/Installers/Validate-ToolCache.ps1 @@ -12,6 +12,21 @@ function GetChildFolders { return Get-ChildItem -Path $Path -Directory -Name } +function Get-ToolcachePackages { + $toolcachePath = Join-Path $env:ROOT_FOLDER "toolcache.json" + return Get-Content -Raw $toolcachePath | ConvertFrom-Json +} + +$packages = (Get-ToolcachePackages).PSObject.Properties | ForEach-Object { + $packageNameParts = $_.Name.Split("-") + $toolName = $packageNameParts[1] + return [PSCustomObject] @{ + ToolName = $packageNameParts[1] + Versions = $_.Value + Arch = $packageNameParts[3] + } +} + function ToolcacheTest { param ( [Parameter(Mandatory = $True)] @@ -22,36 +37,53 @@ function ToolcacheTest { if (Test-Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName") { $description = "" - [array]$versions = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName" - if ($versions.count -gt 0){ - foreach ($version in $versions) + [array]$instaledVersions = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName" + if ($instaledVersions.count -gt 0){ + $softwarePackages = $packages | Where-Object { $_.ToolName -eq $SoftwareName } + foreach($softwarePackage in $softwarePackages) { - $architectures = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$version" - - Write-Host "$SoftwareName version - $version : $([system.String]::Join(",", $architectures))" - - foreach ($arch in $architectures) + foreach ($version in $softwarePackage.Versions) { - $path = "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$version\$arch" - foreach ($test in $ExecTests) - { - if (Test-Path "$path\$test") - { - Write-Host "$SoftwareName($test) $version($arch) is successfully installed:" - Write-Host (& "$path\$test" --version) + $foundVersion = $instaledVersions | where { $_.StartsWith($version) } + + if ($foundVersion -ne $null){ + + $architectures = GetChildFolders -Path "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$foundVersion" + + $softwareArch = $softwarePackage.Arch + + if ($architectures -Contains $softwareArch) { + $path = "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$foundVersion\$softwareArch" + foreach ($test in $ExecTests) + { + if (Test-Path "$path\$test") + { + Write-Host "$SoftwareName($test) $foundVersion($softwareArch) is successfully installed:" + Write-Host (& "$path\$test" --version) + } + else + { + Write-Host "$SoftwareName($test) $foundVersion ($softwareArch) is not installed" + exit 1 + } + } + $description += "_Version:_ $foundVersion ($softwareArch)
" } else { - Write-Host "$SoftwareName($test) $version ($arch) is not installed" + Write-Host "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$foundVersion does not include required architecture" exit 1 } } - - $description += "_Version:_ $version ($arch)
" + else + { + Write-Host "$env:AGENT_TOOLSDIRECTORY\$SoftwareName\$version.* was not found" + exit 1 + } } - } - Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $description + Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $description + } } else { diff --git a/images/win/toolcache.json b/images/win/toolcache.json new file mode 100644 index 000000000..69434da3a --- /dev/null +++ b/images/win/toolcache.json @@ -0,0 +1,17 @@ +{ + "toolcache-python-windows-x64": [ + "2.7", "3.5", "3.6", "3.7", "3.8" + ], + "toolcache-python-windows-x86": [ + "2.7", "3.5", "3.6", "3.7", "3.8" + ], + "toolcache-ruby-windows-x64": [ + "2.4", "2.5", "2.6" + ], + "toolcache-pypy-windows-x86": [ + "2", "3" + ], + "toolcache-boost-windows-x64": [ + "1.69" + ] +} \ No newline at end of file