[Windows] Merge several toolset provisioners into single Configure-Toolset script (#1111)

* Add Configure-Toolset for windows

* Set error action preference to stop

* Move toolcache.json to config folder

* Fix in verbose

* Rename toolsets folder

* Move default version functions from install-toolset to configure-toolset

* Rework Configure-Toolset

* Fix typo and remove empty line

* Fix issues

* Fix parameters in helpers

* Fix helper syntax

* Rename defaultVariable

images/win/Windows2016-Azure.json

@@ -337,12 +337,12 @@
         },
         {
             "type": "file",
-            "source": "{{template_dir}}/toolcache-2016.json",
+            "source": "{{template_dir}}/toolsets/toolcache-2016.json",
             "destination": "{{user `root_folder`}}/toolcache.json"
         },
         {
             "type": "file",
-            "source": "{{template_dir}}/toolset-2016.json",
+            "source": "{{template_dir}}/toolsets/toolset-2016.json",
             "destination": "{{user `toolset_json_path`}}"
         },
         {
@@ -370,16 +370,8 @@
                 "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
             ],
             "scripts":[
-                "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1"
-            ]
-        },
-        {
-            "type": "powershell",
-            "environment_vars":[
-                "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
-            ],
-            "scripts":[
-                "{{ template_dir }}/scripts/Installers/Install-Go.ps1"
+                "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1",
+                "{{ template_dir }}/scripts/Installers/Configure-Toolset.ps1"
             ]
         },
         {
@@ -606,16 +598,6 @@
                 "{{ template_dir }}/scripts/Installers/Install-AzureCosmosDbEmulator.ps1"
             ]
         },
-        {
-            "type": "powershell",
-            "environment_vars": [
-                "ROOT_FOLDER={{user `root_folder`}}",
-                "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
-            ],
-            "scripts":[
-                "{{ template_dir }}/scripts/Installers/Install-Boost.ps1"
-            ]
-        },
         {
             "type": "powershell",
             "scripts":[

images/win/Windows2019-Azure.json

@@ -304,12 +304,12 @@
         },
         {
             "type": "file",
-            "source": "{{template_dir}}/toolcache-2019.json",
+            "source": "{{template_dir}}/toolsets/toolcache-2019.json",
             "destination": "{{user `root_folder`}}/toolcache.json"
         },
         {
             "type": "file",
-            "source": "{{template_dir}}/toolset-2019.json",
+            "source": "{{template_dir}}/toolsets/toolset-2019.json",
             "destination": "{{user `toolset_json_path`}}"
         },
         {
@@ -337,16 +337,8 @@
                 "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
             ],
             "scripts":[
-                "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1"
-            ]
-        },
-        {
-            "type": "powershell",
-            "environment_vars":[
-                "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
-            ],
-            "scripts":[
-                "{{ template_dir }}/scripts/Installers/Install-Go.ps1"
+                "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1",
+                "{{ template_dir }}/scripts/Installers/Configure-Toolset.ps1"
             ]
         },
         {
@@ -567,16 +559,6 @@
                 "{{ template_dir }}/scripts/Installers/Install-AzureCosmosDbEmulator.ps1"
             ]
         },
-        {
-            "type": "powershell",
-            "environment_vars": [
-                "ROOT_FOLDER={{user `root_folder`}}",
-                "TOOLSET_JSON_PATH={{user `toolset_json_path`}}"
-            ],
-            "scripts":[
-                "{{ template_dir }}/scripts/Installers/Install-Boost.ps1"
-            ]
-        },
         {
             "type": "powershell",
             "scripts":[

images/win/scripts/ImageHelpers/ImageHelpers.psm1

@@ -16,6 +16,7 @@ Export-ModuleMember -Function @(
     'Install-VisualStudio'
     'Get-ToolcachePackages'
     'Get-ToolsetContent'
+    'Get-ToolsetToolFullPath'
     'Get-ToolsByName'
     'Stop-SvcWithErrHandling'
     'Set-SvcWithErrHandling'

images/win/scripts/ImageHelpers/InstallHelpers.ps1

@@ -137,7 +137,7 @@ function Stop-SvcWithErrHandling
     .PARAMETER StopOnError
         Switch for stopping the script and exit from PowerShell if one service is absent
     #>
-    param
+    Param
     (
         [Parameter(Mandatory, ValueFromPipeLine = $true)]
         [string] $ServiceName,
@@ -187,7 +187,7 @@ function Set-SvcWithErrHandling
         Hashtable for service arguments
     #>
 
-    param
+    Param
     (
         [Parameter(Mandatory, ValueFromPipeLine = $true)]
         [string] $ServiceName,
@@ -217,7 +217,7 @@ function Set-SvcWithErrHandling
 
 function Start-DownloadWithRetry
 {
-    param
+    Param
     (
         [Parameter(Mandatory)]
         [string] $Url,
@@ -348,17 +348,74 @@ function Get-VSExtensionVersion
     return $packageVersion
 }
 
-function Get-ToolcachePackages {
+function Get-ToolcachePackages
+{
     $toolcachePath = Join-Path $env:ROOT_FOLDER "toolcache.json"
     Get-Content -Raw $toolcachePath | ConvertFrom-Json
 }
 
-function Get-ToolsetContent {
+function Get-ToolsetContent
+{
     $toolsetJson = Get-Content -Path $env:TOOLSET_JSON_PATH -Raw
     ConvertFrom-Json -InputObject $toolsetJson
 }
 
-function Get-ToolsByName {
+function Get-ToolsetToolFullPath
+{
+    <#
+    .DESCRIPTION
+        Function that return full path to specified toolset tool.
+
+    .PARAMETER Name
+        The name of required tool.
+
+    .PARAMETER Version
+        The version of required tool.
+
+    .PARAMETER Arch
+        The architecture of required tool.
+    #>
+
+    Param
+    (
+        [Parameter(Mandatory=$true)]
+        [string] $Name,
+        [Parameter(Mandatory=$true)]
+        [string] $Version,
+        [string] $Arch = "x64"
+    )
+
+    $ToolPath = Join-Path $env:AGENT_TOOLSDIRECTORY $Name
+
+    # Add wildcard if missing
+    if ($Version.Split(".").Length -lt 3) {
+        $Version += ".*"
+    }
+
+    # Check if version folder exists
+    $expectedVersionPath = Join-Path $ToolPath $Version
+    if (-not (Test-Path $expectedVersionPath)) {
+        Write-Host "Expected ${Name} ${Version} folder is not found!"
+        exit 1
+    }
+
+    # Take latest installed version in case if toolset version contains wildcards
+    $foundVersion = Get-Item $expectedVersionPath `
+                    | Sort-Object -Property {[version]$_.name} -Descending `
+                    | Select-Object -First 1
+
+    # Check for required architecture folder
+    $foundVersionArchPath = Join-Path $foundVersion $Arch
+    if (-not (Test-Path $foundVersionArchPath)) {
+        Write-Host "Expected ${Name}(${Arch}) $($foundVersion.name) architecture folder is not found!"
+        exit 1
+    }
+
+    return $foundVersionArchPath
+}
+
+function Get-ToolsByName
+{
     Param
     (
         [Parameter(Mandatory = $True)]
@@ -391,7 +448,7 @@ function Test-IsWin16
 }
 
 function Extract-7Zip {
-    param
+    Param
     (
         [Parameter(Mandatory=$true)]
         [string]$Path,

images/win/scripts/Installers/Configure-Toolset.ps1

@@ -0,0 +1,84 @@
+################################################################################
+##  File:  Configure-Toolset.ps1
+##  Team:  CI-Build
+##  Desc:  Configure Toolset
+################################################################################
+
+Function Set-DefaultVariables
+{
+    param
+    (
+        [Parameter(Mandatory=$true)]
+        [object] $EnvVars,
+        [Parameter(Mandatory=$true)]
+        [string] $ToolVersionPath
+    )
+
+    $templates = $EnvVars.pathTemplates
+    foreach ($template in $templates)
+    {
+        $toolSystemPath = $template -f $ToolVersionPath
+        Add-MachinePathItem -PathItem $toolSystemPath | Out-Null
+    }
+
+    if (-not ([string]::IsNullOrEmpty($EnvVars.defaultVariable)))
+    {
+        setx $toolEnvVars.defaultVariable $ToolVersionPath /M | Out-Null
+    }
+}
+
+$ErrorActionPreference = "Stop"
+
+Import-Module -Name ImageHelpers -Force -DisableNameChecking
+
+# Define executables for cached tools
+$toolsEnvironmentVariables = @{
+    Python = @{
+        pathTemplates = @(
+            "{0}",
+            "{0}\Scripts"
+        )
+    }
+    Boost = @{
+        variableTemplate = "BOOST_ROOT_{0}_{1}_{2}"
+    }
+    go = @{
+        pathTemplates = @(
+            "{0}\bin"
+        )
+        defaultVariable = "GOROOT"
+        variableTemplate = "GOROOT_{0}_{1}_X64"
+    }
+}
+
+$toolsToConfigure = @("Python", "Boost", "Go")
+$tools = Get-ToolsetContent | Select-Object -ExpandProperty toolcache `
+                            | Where-Object { $toolsToConfigure -contains $_.name }
+
+Write-Host "Configure toolset tools environment..."
+foreach ($tool in $tools)
+{
+    $toolEnvVars = $toolsEnvironmentVariables[$tool.name]
+
+    if (-not ([string]::IsNullOrEmpty($toolEnvVars.variableTemplate)))
+    {
+        foreach ($version in $tool.versions)
+        {
+            Write-Host "Set $($tool.name) $version environment variable..."
+
+            $foundVersionArchPath = Get-ToolsetToolFullPath -Name $tool.name -Version $version -Arch $tool.arch
+            $envName = $toolEnvVars.variableTemplate -f $version.Split(".")
+
+            setx $envName $foundVersionArchPath /M | Out-Null
+        }
+    }
+
+    if (-not ([string]::IsNullOrEmpty($tool.default)))
+    {
+        Write-Host "Use $($tool.name) $($tool.default) as a system $($tool.name)..."
+
+        $toolVersionPath = Get-ToolsetToolFullPath -Name $tool.name -Version $tool.default -Arch $tool.arch
+
+        Set-DefaultVariables -ToolVersionPath $toolVersionPath -EnvVars $toolEnvVars
+    }
+}

images/win/scripts/Installers/Install-Boost.ps1

@@ -1,19 +0,0 @@
-################################################################################
-##  File:  Set-BoostRoot.ps1
-##  Team:  CI-Build
-##  Desc:  Install boost using tool cache
-################################################################################
-
-Import-Module -Name ImageHelpers
-
-$SoftwareName = "Boost"
-$BoostDirectory = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath $SoftwareName
-$BoostVersions = (Get-ToolsetContent | Select-Object -ExpandProperty toolcache | Where-Object { $_.Name -eq "Boost"}).Versions
-
-foreach($BoostVersion in $BoostVersions)
-{
-    $BoostInstallationDir = Join-Path -Path $BoostDirectory -ChildPath "$BoostVersion\X86_64"
-
-    $EnvBoostPath = "BOOST_ROOT_{0}" -f ($BoostVersion.Replace('.', '_'))
-    setx $EnvBoostPath $BoostInstallationDir /M | Out-Null
-}

images/win/scripts/Installers/Install-Go.ps1

@@ -1,25 +0,0 @@
-################################################################################
-##  File:  Install-Go.ps1
-##  Desc:  Install Go
-################################################################################
-
-Import-Module -Name ImageHelpers -Force -DisableNameChecking
-
-# Get Go content from toolset
-$goTool = Get-ToolsetContent | Select-Object -ExpandProperty toolcache | Where-Object Name -eq "go"
-
-$toolPath = Join-Path $env:AGENT_TOOLSDIRECTORY $goTool.name
-foreach($goVersion in $goTool.versions)
-{
-    if ($goVersion.Split(".").Length -lt 3) {
-        $goVersion += ".*"
-    }
-    $expectedVersionPath = Join-Path $toolPath $goVersion
-    $foundVersion = Get-Item $expectedVersionPath `
-                    | Sort-Object -Property {[version]$_.name} -Descending `
-                    | Select-Object -First 1
-    # Check for required architecture folder
-    $foundVersionArchPath = Join-Path $foundVersion $goTool.arch
-    $envName = "GOROOT_{0}_{1}_X64" -f $goVersion.split(".")
-    setx $envName "$foundVersionArchPath" /M
-}

images/win/scripts/Installers/Install-Toolset.ps1

@@ -29,47 +29,6 @@ Function Install-Asset {
     Pop-Location
 }
 
-Function Set-DefaultPythonVersion {
-    param(
-        [Parameter(Mandatory=$true)]
-        [object[]] $Toolset
-    )
-
-    $python = $Toolset  | Where-Object { ($_.name -eq "Python") -and ($_.default -ne "") } `
-                        | Select-Object default, arch -First 1
-
-    if ($python.default -ne $null) {
-        $pythonPath = Join-Path $Env:AGENT_TOOLSDIRECTORY "/Python/$($python.default)/$($python.arch)" -Resolve
-
-        Write-Host "Use Python $($python.default) as a system Python"
-        Add-MachinePathItem -PathItem $pythonPath
-        Add-MachinePathItem -PathItem "$pythonPath\Scripts"
-    } else {
-        Write-Host "Default Python version not found in toolset file!"
-    }
-}
-
-Function Set-DefaultGoVersion {
-    param(
-        [Parameter(Mandatory=$true)]
-        [object[]] $Toolset
-    )
-
-    $goToolset = $Toolset  | Where-Object { ($_.name -eq "go") -and ($_.default -ne "") } `
-                           | Select-Object default, arch -First 1
-
-    if ($goToolset.default -ne $null) {
-        $goPath = Join-Path $Env:AGENT_TOOLSDIRECTORY "/go/$($goToolset.default)/$($goToolset.arch)" -Resolve
-
-        Write-Host "Use Go $($goToolset.default) as a system Go"
-        Add-MachinePathItem -PathItem "$goPath\bin" | Out-Null
-        # Set the GOROOT environment variable.
-        setx GOROOT "$goPath" /M | Out-Null
-    } else {
-        Write-Host "Default Go version not found in toolset file!"
-    }
-}
-
 $ErrorActionPreference = "Stop"
 
 Import-Module -Name ImageHelpers -Force
@@ -92,7 +51,7 @@ foreach ($tool in $tools) {
                          | Select-Object -First 1
 
         Write-Host "Installing $($tool.name) $toolVersion $($tool.arch)..."
-        if ($asset -ne $null) {
+        if ($null -ne $asset) {
             Install-Asset -ReleaseAsset $asset
         } else {
             Write-Host "Asset was not found in versions manifest"
@@ -100,7 +59,3 @@ foreach ($tool in $tools) {
         }
     }
 }
-
-# Install default python version
-Set-DefaultPythonVersion -Toolset $tools
-Set-DefaultGoVersion -Toolset $tools

images/win/scripts/Installers/Validate-Toolset.ps1

@@ -75,34 +75,11 @@ $toolsExecutables = @{
 $tools = Get-ToolsetContent | Select-Object -ExpandProperty toolcache
 
 foreach($tool in $tools) {
-    $toolPath = Join-Path $env:AGENT_TOOLSDIRECTORY $tool.name
     # Get executables for current tool
     $toolExecs = $toolsExecutables[$tool.name]
 
     foreach ($version in $tool.versions) {
-        # Add wildcard if missing
-        if ($version.Split(".").Length -lt 3) {
-            $version += ".*"
-        }
-
-        # Check if version folder exists
-        $expectedVersionPath = Join-Path $toolPath $version
-        if (-not (Test-Path $expectedVersionPath)) {
-            Write-Host "Expected $($tool.name) $version folder is not found!"
-            exit 1
-        }
-
-        # Take latest installed version in case if toolset version contains wildcards
-        $foundVersion = Get-Item $expectedVersionPath `
-                        | Sort-Object -Property {[version]$_.name} -Descending `
-                        | Select-Object -First 1
-
-        # Check for required architecture folder
-        $foundVersionArchPath = Join-Path $foundVersion $tool.arch
-        if (-not (Test-Path $foundVersionArchPath)) {
-            Write-Host "Expected $($tool.name)($($tool.arch)) $($foundVersion.name) architecture folder is not found!"
-            exit 1
-        }
+        $foundVersionArchPath = Get-ToolsetToolFullPath -Name $tool.name -Version $version -Arch $tool.arch
 
         if ($toolExecs) {
             Write-Host "Run validation test for $($tool.name)($($tool.arch)) $($foundVersion.name) executables..."