Merge branch 'master' of https://github.com/nikita-bykov/virtual-environments into pipx-windows

2025-12-15 22:26:56 +00:00 · 2020-10-12 11:47:50 +03:00
parent 2977e6c42c 8a499f85a2
commit 71d500699b
20 changed files with 1200 additions and 1107 deletions
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1 @@
-# Do not normalize line endings
-*       -text
+*   text=auto eol=lf
--- a/images/linux/scripts/SoftwareReport/SoftwareReport.Common.psm1
+++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Common.psm1
@@ -245,4 +245,9 @@ function Get-AptPackages {
    $apt = $toolsetJson.apt
    $pkgs = ($apt.common_packages + $apt.cmd_packages | Sort-Object) -join ", "
    return $pkgs
+}
+
+function Get-PipxVersion {
+    $result = (Get-CommandResult "pipx --version").Output
+    return "Pipx $result"
 }
--- a/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1
+++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1
@@ -54,7 +54,8 @@ $markdown += New-MDList -Style Unordered -Lines @(
 )

 $markdown += New-MDHeader "Package Management" -Level 3
-$markdown += New-MDList -Style Unordered -Lines @(
+
+$packageManagementList = @(
        (Get-HomebrewVersion),
        (Get-GemVersion),
        (Get-MinicondaVersion),
@@ -66,6 +67,14 @@ $markdown += New-MDList -Style Unordered -Lines @(
        (Get-VcpkgVersion)
 )

+if (-not (Test-IsUbuntu16)) {
+    $packageManagementList += @(
+        (Get-PipxVersion)
+    )
+}
+
+$markdown += New-MDList -Style Unordered -Lines ($packageManagementList | Sort-Object)
+
 $markdown += New-MDHeader "Project Management" -Level 3
 $markdown += New-MDList -Style Unordered -Lines @(
        (Get-AntVersion),
@@ -115,6 +124,7 @@ $toolsList = @(
    (Get-TerraformVersion),
    (Get-UnZipVersion),
    (Get-WgetVersion),
+    (Get-YamllintVersion),
    (Get-ZipVersion),
    (Get-ZstdVersion)
 )
--- a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1
+++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1
@@ -275,4 +275,8 @@ function Get-RVersion {
 function Get-SphinxVersion {
    $sphinxVersion = searchd -h | Select-Object -First 1 | Take-OutputPart -Part 1 | Take-OutputPart -Part 0 -Delimiter "-"
    return "Sphinx Open Source Search Server $sphinxVersion"
+}
+
+function Get-YamllintVersion {
+    return "$(yamllint --version)"
 }
--- a/images/linux/scripts/installers/pipx-packages.sh
+++ b/images/linux/scripts/installers/pipx-packages.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+################################################################################
+##  File:  pipx-packages.sh
+##  Desc:  Install tools via pipx
+################################################################################
+
+
+export PATH="$PATH:/opt/pipx_bin"
+
+toolset="$INSTALLER_SCRIPT_FOLDER/toolset.json"
+pipx_packages=$(jq -r ".pipx[] .package" $toolset)
+
+for package in $pipx_packages; do
+    python_version=$(jq -r ".pipx[] | select(.package == \"$package\") .python" $toolset)
+    if [ "$python_version" != "null" ]; then
+        python_path="/opt/hostedtoolcache/Python/$python_version*/x64/bin/python$python_version"
+        echo "Install $package into python $python_path"
+        pipx install $package --python $python_path
+    else
+        echo "Install $package into default python"
+        pipx install $package
+    fi
+
+    # Run tests to determine that the software installed as expected
+    cmd=$(jq -r ".pipx[] | select(.package == \"$package\") .cmd" $toolset)
+    if ! command -v $cmd; then
+        echo "$package was not installed"
+        exit 1
+    fi
+done
--- a/images/linux/scripts/installers/python.sh
+++ b/images/linux/scripts/installers/python.sh
@@ -6,18 +6,40 @@

 set -e
 # Source the helpers for use with the script
+source $HELPER_SCRIPTS/etc-environment.sh
 source $HELPER_SCRIPTS/os.sh

 # Install Python, Python 3, pip, pip3
 if isUbuntu16 || isUbuntu18; then
-    apt-get install -y --no-install-recommends python python-dev python-pip python3 python3-dev python3-pip
+    apt-get install -y --no-install-recommends python python-dev python-pip python3 python3-dev python3-pip python3-venv
 fi

 if isUbuntu20; then
-    apt-get install -y --no-install-recommends python3 python3-dev python3-pip
+    apt-get install -y --no-install-recommends python3 python3-dev python3-pip python3-venv
    ln -s /usr/bin/pip3 /usr/bin/pip
 fi

+if isUbuntu18 || isUbuntu20 ; then
+    # Install pipx
+    # Set pipx custom directory
+    export PIPX_BIN_DIR=/opt/pipx_bin
+    export PIPX_HOME=/opt/pipx
+
+    python3 -m pip install pipx
+    python3 -m pipx ensurepath
+
+    # Update /etc/environment
+    setEtcEnvironmentVariable "PIPX_BIN_DIR" $PIPX_BIN_DIR
+    setEtcEnvironmentVariable "PIPX_HOME" $PIPX_HOME
+    prependEtcEnvironmentPath $PIPX_BIN_DIR
+
+    # Test pipx
+    if ! command -v pipx; then
+        echo "pipx was not installed or not found on PATH"
+        exit 1
+    fi
+fi
+
 # Run tests to determine that the software installed as expected
 echo "Testing to make sure that script performed as expected, and basic scenarios work"
 for cmd in python pip python3 pip3; do
--- a/images/linux/toolsets/toolset-1804.json
+++ b/images/linux/toolsets/toolset-1804.json
@@ -164,7 +164,6 @@
            "time",
            "unzip",
            "wget",
-            "yamllint",
            "zip"
        ]
    },
@@ -186,5 +185,15 @@
            "node:12-alpine",
            "ubuntu:14.04"
        ]
-    }
+    },
+    "pipx": [
+        {
+            "package": "yamllint",
+            "cmd": "yamllint"
+        },
+        {
+            "package": "aws-sam-cli",
+            "cmd": "sam"
+        }
+    ]
 }
--- a/images/linux/toolsets/toolset-2004.json
+++ b/images/linux/toolsets/toolset-2004.json
@@ -127,7 +127,6 @@
            "time",
            "unzip",
            "wget",
-            "yamllint",
            "zip"
        ]
    },
@@ -149,5 +148,15 @@
            "node:12-alpine",
            "ubuntu:14.04"
        ]
-    }
+    },
+    "pipx": [
+        {
+            "package": "yamllint",
+            "cmd": "yamllint"
+        },
+        {
+            "package": "aws-sam-cli",
+            "cmd": "sam"
+        }
+    ]
 }
--- a/images/linux/ubuntu1804.json
+++ b/images/linux/ubuntu1804.json
@@ -238,10 +238,10 @@
        {
            "type": "shell",
            "scripts": [
-                "{{template_dir}}/scripts/installers/aws-sam-cli.sh"
+                "{{template_dir}}/scripts/installers/pipx-packages.sh"
            ],
            "environment_vars": [
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}"
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}"
            ],
            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
        },
--- a/images/linux/ubuntu2004.json
+++ b/images/linux/ubuntu2004.json
@@ -240,10 +240,10 @@
        {
            "type": "shell",
            "scripts": [
-                "{{template_dir}}/scripts/installers/aws-sam-cli.sh"
+                "{{template_dir}}/scripts/installers/pipx-packages.sh"
            ],
            "environment_vars": [
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}"
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}"
            ],
            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
        },
--- a/images/macos/provision/configuration/configure-machine.sh
+++ b/images/macos/provision/configuration/configure-machine.sh
@@ -27,7 +27,4 @@ sudo "/Library/Application Support/VMware Tools/vmware-resolutionSet" 1176 885
 # sudo security delete-certificate -Z FF6797793A3CD798DC5B2ABEF56F73EDC9F83A64 /Library/Keychains/System.keychain
 curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer --output $HOME/AppleWWDRCAG3.cer --silent
 sudo security add-trusted-cert -d -r unspecified -k /Library/Keychains/System.keychain $HOME/AppleWWDRCAG3.cer
-rm $HOME/AppleWWDRCAG3.cer
-
-# Disable spotlight indexing to prevent possible high CPU usage after startup
-sudo mdutil -ai off
+rm $HOME/AppleWWDRCAG3.cer
--- a/images/macos/provision/core/edge.sh
+++ b/images/macos/provision/core/edge.sh
@@ -1,7 +1,11 @@
 source ~/utils/utils.sh

 echo "Installing Microsoft Edge..."
+# Workaround to install version 85 since webdriver is broken for 86
+cd "$(brew --repo homebrew/homebrew-cask)"
+git checkout 81f9d08d2b9b7557c0178621078cf59d2c5db2bc
 brew cask install microsoft-edge
+git checkout master

 EDGE_INSTALLATION_PATH="/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"
 EDGE_VERSION=$("$EDGE_INSTALLATION_PATH" --version | cut -d' ' -f 3)
@@ -39,7 +43,7 @@ AUTOUPDATE_START="$HOME/Library/Preferences/com.microsoft.autoupdate2.plist"
 while [ ! -f "$AUTOUPDATE_START" ]
 do
    echo "Wait for MS update automatic installation"
-    sleep 30    
+    sleep 30
 done

 echo "kill autoupdate process"
--- a/images/macos/tests/Xcode.Tests.ps1
+++ b/images/macos/tests/Xcode.Tests.ps1
@@ -56,7 +56,7 @@ Describe "Xcode" {
    }

    Context "XCODE_DEVELOPER_DIR" {
-        $stableXcodeVersions = $testCases | Where-Object { Test-XcodeStableRelease -Version $_ }
+        $stableXcodeVersions = $XCODE_VERSIONS | ForEach-Object { $_.Split("_")[0] } | Where-Object { Test-XcodeStableRelease -Version $_ }
        $majorXcodeVersions = $stableXcodeVersions | ForEach-Object { $_.Split(".")[0] } | Select-Object -Unique
        $testCases = $majorXcodeVersions | ForEach-Object {
            $majorXcodeVersion = $_
--- a/images/macos/toolsets/toolset-10.14.json
+++ b/images/macos/toolsets/toolset-10.14.json
@@ -220,7 +220,8 @@
                "3.5.*",
                "3.6.*",
                "3.7.*",
-                "3.8.*"
+                "3.8.*",
+                "3.9.*"
            ]
        },
        {
--- a/images/macos/toolsets/toolset-10.15.json
+++ b/images/macos/toolsets/toolset-10.15.json
@@ -129,7 +129,8 @@
                "3.5.*",
                "3.6.*",
                "3.7.*",
-                "3.8.*"
+                "3.8.*",
+                "3.9.*"
            ]
        },
        {
--- a/images/macos/toolsets/toolset-11.0.json
+++ b/images/macos/toolsets/toolset-11.0.json
@@ -68,7 +68,8 @@
            "platform" : "darwin",
            "versions": [
                "3.7.*",
-                "3.8.*"
+                "3.8.*",
+                "3.9.*"
            ]
        },
        {
--- a/images/win/Windows2016-Readme.md
+++ b/images/win/Windows2016-Readme.md
--- a/images/win/Windows2019-Readme.md
+++ b/images/win/Windows2019-Readme.md
--- a/images/win/scripts/Installers/Install-Git.ps1
+++ b/images/win/scripts/Installers/Install-Git.ps1
@@ -38,8 +38,11 @@ Choco-Install -PackageName hub

 Add-MachinePathItem "C:\Program Files\Git\bin"

-# Add well-known SSH host keys to ssh_known_hosts
+if (Test-IsWin16) {
+    $env:Path += ";$env:ProgramFiles\Git\usr\bin\"
+}

+# Add well-known SSH host keys to ssh_known_hosts
 ssh-keyscan -t rsa github.com >> "C:\Program Files\Git\etc\ssh\ssh_known_hosts"
 ssh-keyscan -t rsa ssh.dev.azure.com >> "C:\Program Files\Git\etc\ssh\ssh_known_hosts"

--- a/images/win/scripts/Installers/Install-Haskell.ps1
+++ b/images/win/scripts/Installers/Install-Haskell.ps1
@@ -4,7 +4,7 @@
 ################################################################################

 # Get 3 latest versions of GHC
-[Version[]] $ChocoVersionsOutput = & choco search ghc --allversions --limit-output | Where-Object { $_.StartsWith("ghc|") } | ForEach-Object { $_.TrimStart("ghc|") }
+[Version[]] $ChocoVersionsOutput = & choco search ghc --allversions | Where-Object { $_.StartsWith("ghc ") -and $_ -match "Approved"} | ForEach-Object { [regex]::matches($_, "\d+(\.\d+){2,}").value }
 $MajorMinorGroups = $ChocoVersionsOutput | Sort-Object -Descending | Group-Object { $_.ToString(2) } | Select-Object -First 3
 $VersionsList = $MajorMinorGroups | ForEach-Object { $_.Group | Select-Object -First 1 } | Sort-Object