Merge branch 'master' of https://github.com/nikita-bykov/virtual-environments into swift-support

.github/pull_request_template.md

@@ -0,0 +1,12 @@
+# Description
+New tool, Bug fixing, or Improvement?  
+Please include a summary of the change and which issue is fixed. Also include relevant motivation and context.  
+**For new tools, please provide total size and installation time.**
+
+#### Related issue:
+
+## Check list
+- [ ] Related issue / work item is attached
+- [ ] Tests are written (if applicable)
+- [ ] Documentation is updated (if applicable)
+- [ ] Changes are tested and related VM images are successfully generated

images/linux/scripts/installers/clang.sh

@@ -12,13 +12,11 @@ function InstallClang {
     version=$1
 
     echo "Installing clang-$version..."
-    # Clang 6.0 is not supported by automatic installation script (`llvm.sh`)
+    if [[ $version =~ 9 ]]; then
-    # Thus we have to install it explicitly
-    if [[ $version == 6* ]]; then
-        apt-get install -y "clang-$version" "lldb-$version" "lld-$version"
-    else
         ./llvm.sh $version
         apt-get install -y "clang-format-$version"
+    else
+        apt-get install -y "clang-$version" "lldb-$version" "lld-$version" "clang-format-$version"
     fi
 
     # Run tests to determine that the software installed as expected
@@ -35,11 +33,6 @@ function InstallClang {
     DocumentInstalledItem "Clang $version ($(clang-$version --version | head -n 1 | cut -d ' ' -f 3 | cut -d '-' -f 1))"
 }
 
-# Install Clang compiler
-wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
-apt-add-repository "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-6.0 main"
-apt-get update -y
-
 # Download script for automatic installation
 wget https://apt.llvm.org/llvm.sh
 chmod +x llvm.sh

images/linux/scripts/installers/rust.sh

@@ -12,16 +12,14 @@ set -e
 export RUSTUP_HOME=/usr/share/rust/.rustup
 export CARGO_HOME=/usr/share/rust/.cargo
 
-curl https://sh.rustup.rs -sSf | sh -s -- -y
+curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain=stable --profile=minimal
 
 # Initialize environment variables
 source $CARGO_HOME/env
 
 # Install common tools
-rustup component add rustfmt
+rustup component add rustfmt clippy
-rustup component add clippy
+cargo install bindgen cbindgen
-cargo install bindgen
-cargo install cbindgen
 
 echo "Test installation of the Rust toochain"
 

images/win/Windows2016-Azure.json

@@ -15,6 +15,7 @@
         "virtual_network_subnet_name": "{{env `VNET_SUBNET`}}",
         "private_virtual_network_with_public_ip": "{{env `PRIVATE_VIRTUAL_NETWORK_WITH_PUBLIC_IP`}}",
         "vm_size": "Standard_DS4_v2",
+        "run_scan_antivirus": "false",
 
         "root_folder": "C:",
         "image_folder": "C:\\image",
@@ -30,8 +31,7 @@
         "image_os": "win16",
         "github_feed_token": "{{env `GITHUB_FEED_TOKEN`}}",
         "go_versions": "1.9, 1.10, 1.11, 1.12, 1.13, 1.14",
-        "go_default": "1.14",
+        "go_default": "1.14"
-        "boost_default": "1.69.0"
     },
     "sensitive-variables": ["install_password", "ssh_password", "client_secret", "github_feed_token"],
     "builders": [
@@ -101,7 +101,8 @@
             ],
             "scripts":[
                 "{{ template_dir }}/scripts/Installers/Windows2016/Initialize-VM.ps1"
-            ]
+            ],
+            "execution_policy": "unrestricted"
         },
         {
             "type": "powershell",
@@ -517,8 +518,7 @@
         {
             "type": "powershell",
             "environment_vars": [
-                "ROOT_FOLDER={{user `root_folder`}}",
+                "ROOT_FOLDER={{user `root_folder`}}"
-                "BOOST_DEFAULT={{user `boost_default`}}"
             ],
             "scripts":[
                 "{{ template_dir }}/scripts/Installers/Install-Boost.ps1"
@@ -652,8 +652,7 @@
         {
             "type": "powershell",
             "environment_vars": [
-                "ROOT_FOLDER={{user `root_folder`}}",
+                "ROOT_FOLDER={{user `root_folder`}}"
-                "BOOST_DEFAULT={{user `boost_default`}}"
             ],
             "scripts":[
                 "{{ template_dir }}/scripts/Installers/Validate-Boost.ps1"
@@ -875,8 +874,17 @@
         },
         {
             "type": "powershell",
+            "environment_vars":[
+                "RUN_SCAN_ANTIVIRUS={{user `run_scan_antivirus`}}"
+            ],
             "scripts":[
-                "{{ template_dir }}/scripts/Installers/Windows2016/Run-Antivirus.ps1"
+                "{{ template_dir }}/scripts/Installers/Run-Antivirus.ps1"
+            ]
+        },
+        {
+            "type": "powershell",
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Configure-Antivirus.ps1"
             ]
         },
         {

images/win/Windows2019-Azure.json

@@ -15,6 +15,7 @@
         "virtual_network_subnet_name": "{{env `VNET_SUBNET`}}",
         "private_virtual_network_with_public_ip": "{{env `PRIVATE_VIRTUAL_NETWORK_WITH_PUBLIC_IP`}}",
         "vm_size": "Standard_DS4_v2",
+        "run_scan_antivirus": "false",
 
         "root_folder": "C:",
         "image_folder": "C:\\image",
@@ -30,8 +31,7 @@
         "image_os": "win19",
         "github_feed_token": "{{env `GITHUB_FEED_TOKEN`}}",
         "go_versions": "1.9, 1.10, 1.11, 1.12, 1.13, 1.14",
-        "go_default": "1.14",
+        "go_default": "1.14"
-        "boost_default": "1.72.0"
     },
     "sensitive-variables": ["install_password", "ssh_password", "client_secret", "github_feed_token"],
     "builders": [
@@ -101,7 +101,8 @@
             ],
             "scripts":[
                 "{{ template_dir }}/scripts/Installers/Windows2019/Initialize-VM.ps1"
-            ]
+            ],
+            "execution_policy": "unrestricted"
         },
         {
             "type": "powershell",
@@ -498,8 +499,7 @@
         {
             "type": "powershell",
             "environment_vars": [
-                "ROOT_FOLDER={{user `root_folder`}}",
+                "ROOT_FOLDER={{user `root_folder`}}"
-                "BOOST_DEFAULT={{user `boost_default`}}"
             ],
             "scripts":[
                 "{{ template_dir }}/scripts/Installers/Install-Boost.ps1"
@@ -639,8 +639,7 @@
         {
             "type": "powershell",
             "environment_vars": [
-                "ROOT_FOLDER={{user `root_folder`}}",
+                "ROOT_FOLDER={{user `root_folder`}}"
-                "BOOST_DEFAULT={{user `boost_default`}}"
             ],
             "scripts":[
                 "{{ template_dir }}/scripts/Installers/Validate-Boost.ps1"
@@ -868,8 +867,17 @@
         },
         {
             "type": "powershell",
+            "environment_vars":[
+                "RUN_SCAN_ANTIVIRUS={{user `run_scan_antivirus`}}"
+            ],
             "scripts":[
-                "{{ template_dir }}/scripts/Installers/Windows2019/Run-Antivirus.ps1"
+                "{{ template_dir }}/scripts/Installers/Run-Antivirus.ps1"
+            ]
+        },
+        {
+            "type": "powershell",
+            "scripts":[
+                "{{ template_dir }}/scripts/Installers/Configure-Antivirus.ps1"
             ]
         },
         {

images/win/scripts/Installers/Configure-Antivirus.ps1

@@ -0,0 +1,2 @@
+Write-Host "Set antivirus parameters"
+Set-MpPreference -ScanAvgCPULoadFactor 5 -ExclusionPath "D:\", "C:\"

images/win/scripts/Installers/Install-Boost.ps1

@@ -9,7 +9,6 @@ Import-Module -Name ImageHelpers
 $SoftwareName = "Boost"
 $BoostDirectory = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath $SoftwareName
 $BoostVersions = (Get-ToolsByName -SoftwareName $SoftwareName).Versions | Foreach-Object {"{0}.0" -f $_}
-$BoostDefault = $env:BOOST_DEFAULT
 
 foreach($BoostVersion in $BoostVersions)
 {
@@ -17,11 +16,4 @@ foreach($BoostVersion in $BoostVersions)
 
     $EnvBoostPath = "BOOST_ROOT_{0}" -f ($BoostVersion.Replace('.', '_'))
     setx $EnvBoostPath $BoostInstallationDir /M | Out-Null
-
-    if ($BoostVersion -eq $BoostDefault)
-    {
-        Write-Host "Adding Boost $BoostVersion to the path..."
-        # Add the Boost binaries to the path
-        Add-MachinePathItem $BoostInstallationDir | Out-Null
-    }
 }

images/win/scripts/Installers/Install-PHP.ps1

@@ -10,8 +10,11 @@ Import-Module -Name ImageHelpers
 $installDir = "c:\tools\php"
 choco install php -y --force --params "/InstallDir:$installDir"
 
-# update path to extensions and enable curl and mbstring extensions
+# Install latest Composer in chocolatey
-((Get-Content -path $installDir\php.ini -Raw) -replace ';extension=curl','extension=curl' -replace ';extension=mbstring','extension=mbstring' -replace ';extension_dir = "ext"','extension_dir = "ext"') | Set-Content -Path $installDir\php.ini
+choco install composer --ia "/DEV=$installDir /PHP=$installDir"
+
+# update path to extensions and enable curl and mbstring extensions, and enable php openssl extensions.
+((Get-Content -path $installDir\php.ini -Raw) -replace ';extension=curl','extension=curl' -replace ';extension=mbstring','extension=mbstring' -replace ';extension_dir = "ext"','extension_dir = "ext"' -replace 'extension=";php_openssl.dll"','extension_dir = "php_openssl.dll"') | Set-Content -Path $installDir\php.ini
 
 # Set the PHPROOT environment variable.
 setx PHPROOT $installDir /M

images/win/scripts/Installers/Install-Rust.ps1

@@ -14,7 +14,7 @@ $env:CARGO_HOME="C:\Rust\.cargo"
 Invoke-WebRequest -UseBasicParsing -Uri "https://win.rustup.rs/x86_64" -OutFile rustup-init.exe
 
 # Install Rust by running rustup-init.exe (disabling the confirmation prompt with -y)
-.\rustup-init.exe -y
+.\rustup-init.exe -y --default-toolchain=stable --profile=minimal
 
 # Delete rustup-init.exe when it's no longer needed
 Remove-Item -Path .\rustup-init.exe
@@ -24,10 +24,8 @@ Add-MachinePathItem "$env:CARGO_HOME\bin"
 $env:Path = Get-MachinePath
 
 # Install common tools
-rustup component add rustfmt
+rustup component add rustfmt clippy
-rustup component add clippy
+cargo install bindgen cbindgen
-cargo install bindgen
-cargo install cbindgen
 
 # Run script at startup for all users
 $cmdRustSymScript = @"

images/win/scripts/Installers/Run-Antivirus.ps1

@@ -0,0 +1,30 @@
+################################################################################
+##  File:  Run-Antivirus.ps1
+##  Desc:  Run a full antivirus scan.
+##         Run right after cleanup before we sysprep
+################################################################################
+
+if ($env:run_scan_antivirus -eq $true) {
+    try {
+        Update-MpSignature
+    }
+    catch {
+        Write-Host "Some error was found"
+        Write-Host $_
+    }
+
+    Write-Host "Make sure windefend is going to start"
+    Start-Service windefend -ErrorAction Continue
+    Write-Host "Waiting for windefend to report as running"
+    $service = Get-Service "Windefend"
+    $service.WaitForStatus("Running","00:10:00")
+
+    Write-Host "Run antivirus"
+    # Tell Defender to use 100% of the CPU during the scan
+    Set-MpPreference -ScanAvgCPULoadFactor 100
+    # Full Scan
+    Start-Process -FilePath "C:\Program Files\Windows Defender\MpCmdRun.exe" -ArgumentList ("-Scan","-ScanType", 2) -Wait
+}
+else {
+    Write-Host "Scanning process has been skipped"
+}

images/win/scripts/Installers/Validate-Boost.ps1

@@ -18,7 +18,6 @@ function Validate-BoostVersion
     if (Test-Path "$ReleasePath\b2.exe")
     {
         Write-Host "Boost.Build $BoostRelease is successfully installed"
-
         return
     }
 
@@ -26,17 +25,6 @@ function Validate-BoostVersion
     exit 1
 }
 
-# Verify that Boost is on the path
-if (Get-Command -Name 'b2')
-{
-    Write-Host "Boost is on the path"
-}
-else
-{
-    Write-Host "Boost is not on the path"
-    exit 1
-}
-
 # Adding description of the software to Markdown
 $tmplMark = @"
 #### {0} [{2}]
@@ -46,14 +34,6 @@ _Environment:_
 
 "@
 
-$tmplMarkRoot = @"
-#### {0} [{2}]
-
-_Environment:_
-* PATH: contains the location of Boost version {0}
-* {1}: root directory of the Boost version {0} installation
-"@
-
 $Description = New-Object System.Text.StringBuilder
 $SoftwareName = 'Boost'
 $BoostRootDirectory = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath $SoftwareName
@@ -68,16 +48,9 @@ foreach ($BoostTool in $BoostTools)
         Validate-BoostVersion -BoostRootPath $BoostRootDirectory -BoostRelease $BoostVersion
         $BoostVersionTag = "BOOST_ROOT_{0}" -f $BoostVersion.Replace('.', '_')
 
-        if($BoostVersion -eq $env:BOOST_DEFAULT)
-        {
-            $null = $Description.AppendLine(($tmplMarkRoot -f $BoostVersion, $BoostVersionTag, $BoostToolsetName))
-        }
-        else
-        {
         $null = $Description.AppendLine(($tmplMark -f $BoostVersion, $BoostVersionTag, $BoostToolsetName))
     }
 }
-}
 
 $CMakeFindBoostInfo = @"
 

images/win/scripts/Installers/Validate-NSIS.ps1

@@ -28,5 +28,4 @@ $Description = @"
 _Version:_ $Version<br/>
 "@
 
-#Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $Description
+Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $Description
-Write-Host $description

images/win/scripts/Installers/Validate-PHP.ps1

@@ -33,6 +33,32 @@ else
     exit 1
 }
 
+# Verify that composer.exe is on the path
+if(Get-Command -Name 'composer')
+{
+    Write-Host "$(composer --version) is on the path."
+}
+else
+{
+    Write-Host "composer is not on the path."
+    exit 1
+}
+
+# Get the composer version.
+$composerVersion = $(composer --version)
+
+# Add composer version details in Markdown
+$SoftwareName = "Composer"
+$Description = @"
+#### $composerVersion
+
+_Environment:_
+* PATH: contains the location of composer.exe version $composerVersion
+* PHPROOT: root directory of the Composer $composerVersion installation
+"@
+
+Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $Description
+
 # Get available versions of PHP
 $phpVersionOnPath = Get-PHPVersion -phpRootPath "C:\tools\php72"
 

images/win/scripts/Installers/Windows2016/Run-Antivirus.ps1

@@ -1,19 +0,0 @@
-################################################################################
-##  File:  Run-Antivirus.ps1
-##  Desc:  Run a full antivirus scan.
-##         Run right after cleanup before we sysprep
-################################################################################
-
-Write-Host "Run antivirus"
-Push-Location "C:\Program Files\Windows Defender"
-
-# Tell Defender to use 100% of the CPU during the scan
-Set-MpPreference -ScanAvgCPULoadFactor 100
-
-# Full Scan
-.\MpCmdRun.exe -Scan -ScanType 2
-Pop-Location
-
-Write-Host "Set antivirus parmeters"
-Set-MpPreference -ScanAvgCPULoadFactor 5 `
-                 -ExclusionPath "D:\", "C:\"

images/win/scripts/Installers/Windows2019/Install-VS2019.ps1

@@ -121,6 +121,7 @@ $WorkLoads = '--allWorkloads --includeRecommended ' + `
               '--add Microsoft.VisualStudio.Component.VC.v141.MFC.ARM.Spectre ' + `
               '--add Microsoft.VisualStudio.Component.VC.v141.MFC.ARM64.Spectre ' + `
               '--add Microsoft.VisualStudio.Component.VC.v141.MFC.Spectre ' + `
+              '--add Microsoft.VisualStudio.Component.Windows10SDK.14393 ' + `
               '--add Microsoft.VisualStudio.Component.Windows10SDK.16299 ' + `
               '--add Microsoft.VisualStudio.Component.Windows10SDK.17134 ' + `
               '--add Microsoft.VisualStudio.Component.Windows10SDK.17763 ' + `

images/win/scripts/Installers/Windows2019/Run-Antivirus.ps1

@@ -1,27 +0,0 @@
-################################################################################
-##  File:  Run-Antivirus.ps1
-##  Desc:  Run a full antivirus scan.
-##         Run right after cleanup before we sysprep
-################################################################################
-
-Write-Host "Make sure windefend is going to start"
-Start-Service windefend -ErrorAction Continue
-
-Write-Host "Waiting for windefend to report as running"
-$service = Get-Service "Windefend"
-$service.WaitForStatus("Running","00:10:00")
-
-Write-Host "Run antivirus"
-Push-Location "C:\Program Files\Windows Defender"
-
-# Tell Defender to use 100% of the CPU during the scan
-Set-MpPreference -ScanAvgCPULoadFactor 100
-
-# Full Scan
-.\MpCmdRun.exe -Scan -ScanType 2
-Pop-Location
-
-Update-MpSignature
-Write-Host "Set antivirus parmeters"
-Set-MpPreference -ScanAvgCPULoadFactor 5 `
-                 -ExclusionPath "D:\", "C:\"