Merge branch 'master' of https://github.com/nikita-bykov/virtual-environments into pipx-windows

.gitattributes

@@ -1,2 +1 @@
-# Do not normalize line endings
+*   text=auto eol=lf
-*       -text

images/linux/scripts/SoftwareReport/SoftwareReport.Common.psm1

@@ -245,4 +245,9 @@ function Get-AptPackages {
     $apt = $toolsetJson.apt
     $pkgs = ($apt.common_packages + $apt.cmd_packages | Sort-Object) -join ", "
     return $pkgs
+}
+
+function Get-PipxVersion {
+    $result = (Get-CommandResult "pipx --version").Output
+    return "Pipx $result"
 }

images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1

@@ -54,7 +54,8 @@ $markdown += New-MDList -Style Unordered -Lines @(
 )
 
 $markdown += New-MDHeader "Package Management" -Level 3
-$markdown += New-MDList -Style Unordered -Lines @(
+
+$packageManagementList = @(
         (Get-HomebrewVersion),
         (Get-GemVersion),
         (Get-MinicondaVersion),
@@ -66,6 +67,14 @@ $markdown += New-MDList -Style Unordered -Lines @(
         (Get-VcpkgVersion)
 )
 
+if (-not (Test-IsUbuntu16)) {
+    $packageManagementList += @(
+        (Get-PipxVersion)
+    )
+}
+
+$markdown += New-MDList -Style Unordered -Lines ($packageManagementList | Sort-Object)
+
 $markdown += New-MDHeader "Project Management" -Level 3
 $markdown += New-MDList -Style Unordered -Lines @(
         (Get-AntVersion),
@@ -115,6 +124,7 @@ $toolsList = @(
     (Get-TerraformVersion),
     (Get-UnZipVersion),
     (Get-WgetVersion),
+    (Get-YamllintVersion),
     (Get-ZipVersion),
     (Get-ZstdVersion)
 )

images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1

@@ -275,4 +275,8 @@ function Get-RVersion {
 function Get-SphinxVersion {
     $sphinxVersion = searchd -h | Select-Object -First 1 | Take-OutputPart -Part 1 | Take-OutputPart -Part 0 -Delimiter "-"
     return "Sphinx Open Source Search Server $sphinxVersion"
+}
+
+function Get-YamllintVersion {
+    return "$(yamllint --version)"
 }

images/linux/scripts/installers/pipx-packages.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+################################################################################
+##  File:  pipx-packages.sh
+##  Desc:  Install tools via pipx
+################################################################################
+
+
+export PATH="$PATH:/opt/pipx_bin"
+
+toolset="$INSTALLER_SCRIPT_FOLDER/toolset.json"
+pipx_packages=$(jq -r ".pipx[] .package" $toolset)
+
+for package in $pipx_packages; do
+    python_version=$(jq -r ".pipx[] | select(.package == \"$package\") .python" $toolset)
+    if [ "$python_version" != "null" ]; then
+        python_path="/opt/hostedtoolcache/Python/$python_version*/x64/bin/python$python_version"
+        echo "Install $package into python $python_path"
+        pipx install $package --python $python_path
+    else
+        echo "Install $package into default python"
+        pipx install $package
+    fi
+
+    # Run tests to determine that the software installed as expected
+    cmd=$(jq -r ".pipx[] | select(.package == \"$package\") .cmd" $toolset)
+    if ! command -v $cmd; then
+        echo "$package was not installed"
+        exit 1
+    fi
+done

images/linux/scripts/installers/python.sh

@@ -6,18 +6,40 @@
 
 set -e
 # Source the helpers for use with the script
+source $HELPER_SCRIPTS/etc-environment.sh
 source $HELPER_SCRIPTS/os.sh
 
 # Install Python, Python 3, pip, pip3
 if isUbuntu16 || isUbuntu18; then
-    apt-get install -y --no-install-recommends python python-dev python-pip python3 python3-dev python3-pip
+    apt-get install -y --no-install-recommends python python-dev python-pip python3 python3-dev python3-pip python3-venv
 fi
 
 if isUbuntu20; then
-    apt-get install -y --no-install-recommends python3 python3-dev python3-pip
+    apt-get install -y --no-install-recommends python3 python3-dev python3-pip python3-venv
     ln -s /usr/bin/pip3 /usr/bin/pip
 fi
 
+if isUbuntu18 || isUbuntu20 ; then
+    # Install pipx
+    # Set pipx custom directory
+    export PIPX_BIN_DIR=/opt/pipx_bin
+    export PIPX_HOME=/opt/pipx
+
+    python3 -m pip install pipx
+    python3 -m pipx ensurepath
+
+    # Update /etc/environment
+    setEtcEnvironmentVariable "PIPX_BIN_DIR" $PIPX_BIN_DIR
+    setEtcEnvironmentVariable "PIPX_HOME" $PIPX_HOME
+    prependEtcEnvironmentPath $PIPX_BIN_DIR
+
+    # Test pipx
+    if ! command -v pipx; then
+        echo "pipx was not installed or not found on PATH"
+        exit 1
+    fi
+fi
+
 # Run tests to determine that the software installed as expected
 echo "Testing to make sure that script performed as expected, and basic scenarios work"
 for cmd in python pip python3 pip3; do

images/linux/toolsets/toolset-1804.json

@@ -164,7 +164,6 @@
             "time",
             "unzip",
             "wget",
-            "yamllint",
             "zip"
         ]
     },
@@ -186,5 +185,15 @@
             "node:12-alpine",
             "ubuntu:14.04"
         ]
-    }
+    },
+    "pipx": [
+        {
+            "package": "yamllint",
+            "cmd": "yamllint"
+        },
+        {
+            "package": "aws-sam-cli",
+            "cmd": "sam"
+        }
+    ]
 }

images/linux/toolsets/toolset-2004.json

@@ -127,7 +127,6 @@
             "time",
             "unzip",
             "wget",
-            "yamllint",
             "zip"
         ]
     },
@@ -149,5 +148,15 @@
             "node:12-alpine",
             "ubuntu:14.04"
         ]
-    }
+    },
+    "pipx": [
+        {
+            "package": "yamllint",
+            "cmd": "yamllint"
+        },
+        {
+            "package": "aws-sam-cli",
+            "cmd": "sam"
+        }
+    ]
 }

images/linux/ubuntu1804.json

@@ -238,10 +238,10 @@
         {
             "type": "shell",
             "scripts": [
-                "{{template_dir}}/scripts/installers/aws-sam-cli.sh"
+                "{{template_dir}}/scripts/installers/pipx-packages.sh"
             ],
             "environment_vars": [
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}"
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },

images/linux/ubuntu2004.json

@@ -240,10 +240,10 @@
         {
             "type": "shell",
             "scripts": [
-                "{{template_dir}}/scripts/installers/aws-sam-cli.sh"
+                "{{template_dir}}/scripts/installers/pipx-packages.sh"
             ],
             "environment_vars": [
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}"
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },

images/macos/provision/configuration/configure-machine.sh

@@ -27,7 +27,4 @@ sudo "/Library/Application Support/VMware Tools/vmware-resolutionSet" 1176 885
 # sudo security delete-certificate -Z FF6797793A3CD798DC5B2ABEF56F73EDC9F83A64 /Library/Keychains/System.keychain
 curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer --output $HOME/AppleWWDRCAG3.cer --silent
 sudo security add-trusted-cert -d -r unspecified -k /Library/Keychains/System.keychain $HOME/AppleWWDRCAG3.cer
-rm $HOME/AppleWWDRCAG3.cer
+rm $HOME/AppleWWDRCAG3.cer
-
-# Disable spotlight indexing to prevent possible high CPU usage after startup
-sudo mdutil -ai off

images/macos/provision/core/edge.sh

@@ -1,7 +1,11 @@
 source ~/utils/utils.sh
 
 echo "Installing Microsoft Edge..."
+# Workaround to install version 85 since webdriver is broken for 86
+cd "$(brew --repo homebrew/homebrew-cask)"
+git checkout 81f9d08d2b9b7557c0178621078cf59d2c5db2bc
 brew cask install microsoft-edge
+git checkout master
 
 EDGE_INSTALLATION_PATH="/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"
 EDGE_VERSION=$("$EDGE_INSTALLATION_PATH" --version | cut -d' ' -f 3)
@@ -39,7 +43,7 @@ AUTOUPDATE_START="$HOME/Library/Preferences/com.microsoft.autoupdate2.plist"
 while [ ! -f "$AUTOUPDATE_START" ]
 do
     echo "Wait for MS update automatic installation"
-    sleep 30    
+    sleep 30
 done
 
 echo "kill autoupdate process"

images/macos/tests/Xcode.Tests.ps1

@@ -56,7 +56,7 @@ Describe "Xcode" {
     }
 
     Context "XCODE_DEVELOPER_DIR" {
-        $stableXcodeVersions = $testCases | Where-Object { Test-XcodeStableRelease -Version $_ }
+        $stableXcodeVersions = $XCODE_VERSIONS | ForEach-Object { $_.Split("_")[0] } | Where-Object { Test-XcodeStableRelease -Version $_ }
         $majorXcodeVersions = $stableXcodeVersions | ForEach-Object { $_.Split(".")[0] } | Select-Object -Unique
         $testCases = $majorXcodeVersions | ForEach-Object {
             $majorXcodeVersion = $_

images/macos/toolsets/toolset-10.14.json

@@ -220,7 +220,8 @@
                 "3.5.*",
                 "3.6.*",
                 "3.7.*",
-                "3.8.*"
+                "3.8.*",
+                "3.9.*"
             ]
         },
         {

images/macos/toolsets/toolset-10.15.json

@@ -129,7 +129,8 @@
                 "3.5.*",
                 "3.6.*",
                 "3.7.*",
-                "3.8.*"
+                "3.8.*",
+                "3.9.*"
             ]
         },
         {

images/macos/toolsets/toolset-11.0.json

@@ -68,7 +68,8 @@
             "platform" : "darwin",
             "versions": [
                 "3.7.*",
-                "3.8.*"
+                "3.8.*",
+                "3.9.*"
             ]
         },
         {

images/win/scripts/Installers/Install-Git.ps1

@@ -38,8 +38,11 @@ Choco-Install -PackageName hub
 
 Add-MachinePathItem "C:\Program Files\Git\bin"
 
-# Add well-known SSH host keys to ssh_known_hosts
+if (Test-IsWin16) {
+    $env:Path += ";$env:ProgramFiles\Git\usr\bin\"
+}
 
+# Add well-known SSH host keys to ssh_known_hosts
 ssh-keyscan -t rsa github.com >> "C:\Program Files\Git\etc\ssh\ssh_known_hosts"
 ssh-keyscan -t rsa ssh.dev.azure.com >> "C:\Program Files\Git\etc\ssh\ssh_known_hosts"
 

images/win/scripts/Installers/Install-Haskell.ps1

@@ -4,7 +4,7 @@
 ################################################################################
 
 # Get 3 latest versions of GHC
-[Version[]] $ChocoVersionsOutput = & choco search ghc --allversions --limit-output | Where-Object { $_.StartsWith("ghc|") } | ForEach-Object { $_.TrimStart("ghc|") }
+[Version[]] $ChocoVersionsOutput = & choco search ghc --allversions | Where-Object { $_.StartsWith("ghc ") -and $_ -match "Approved"} | ForEach-Object { [regex]::matches($_, "\d+(\.\d+){2,}").value }
 $MajorMinorGroups = $ChocoVersionsOutput | Sort-Object -Descending | Group-Object { $_.ToString(2) } | Select-Object -First 3
 $VersionsList = $MajorMinorGroups | ForEach-Object { $_.Group | Select-Object -First 1 } | Sort-Object