Merge branch 'main' into v-dmshib/add-python3.9-win

2026-01-01 23:23:56 +08:00 · 2020-10-09 14:38:03 +03:00
parent 310281c804 65f9b7ff6b
commit cc2314394b
14 changed files with 107 additions and 19 deletions
--- a/images/linux/scripts/SoftwareReport/SoftwareReport.Common.psm1
+++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Common.psm1
@@ -245,4 +245,8 @@ function Get-AptPackages {
    $apt = $toolsetJson.apt
    $pkgs = ($apt.common_packages + $apt.cmd_packages | Sort-Object) -join ", "
    return $pkgs
+}
+
+function Get-PipxVersion {
+    return "Pipx $(pipx --version 2> $null)"
 }
--- a/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1
+++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1
@@ -54,7 +54,8 @@ $markdown += New-MDList -Style Unordered -Lines @(
 )

 $markdown += New-MDHeader "Package Management" -Level 3
-$markdown += New-MDList -Style Unordered -Lines @(
+
+$packageManagementList = @(
        (Get-HomebrewVersion),
        (Get-GemVersion),
        (Get-MinicondaVersion),
@@ -66,6 +67,14 @@ $markdown += New-MDList -Style Unordered -Lines @(
        (Get-VcpkgVersion)
 )

+if (-not (Test-IsUbuntu16)) {
+    $packageManagementList += @(
+        (Get-PipxVersion)
+    )
+}
+
+$markdown += New-MDList -Style Unordered -Lines ($packageManagementList | Sort-Object)
+
 $markdown += New-MDHeader "Project Management" -Level 3
 $markdown += New-MDList -Style Unordered -Lines @(
        (Get-AntVersion),
@@ -115,6 +124,7 @@ $toolsList = @(
    (Get-TerraformVersion),
    (Get-UnZipVersion),
    (Get-WgetVersion),
+    (Get-YamllintVersion),
    (Get-ZipVersion),
    (Get-ZstdVersion)
 )
--- a/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1
+++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Tools.psm1
@@ -275,4 +275,8 @@ function Get-RVersion {
 function Get-SphinxVersion {
    $sphinxVersion = searchd -h | Select-Object -First 1 | Take-OutputPart -Part 1 | Take-OutputPart -Part 0 -Delimiter "-"
    return "Sphinx Open Source Search Server $sphinxVersion"
+}
+
+function Get-YamllintVersion {
+    return "$(yamllint --version)"
 }
--- a/images/linux/scripts/installers/pipx-packages.sh
+++ b/images/linux/scripts/installers/pipx-packages.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+################################################################################
+##  File:  pipx-packages.sh
+##  Desc:  Install tools via pipx
+################################################################################
+
+
+export PATH="$PATH:/opt/pipx_bin"
+
+toolset="$INSTALLER_SCRIPT_FOLDER/toolset.json"
+pipx_packages=$(jq -r ".pipx[] .package" $toolset)
+
+for package in $pipx_packages; do
+    python_version=$(jq -r ".pipx[] | select(.package == \"$package\") .python" $toolset)
+    if [ "$python_version" != "null" ]; then
+        python_path="/opt/hostedtoolcache/Python/$python_version*/x64/bin/python$python_version"
+        echo "Install $package into python $python_path"
+        pipx install $package --python $python_path
+    else
+        echo "Install $package into default python"
+        pipx install $package
+    fi
+
+    # Run tests to determine that the software installed as expected
+    cmd=$(jq -r ".pipx[] | select(.package == \"$package\") .cmd" $toolset)
+    if ! command -v $cmd; then
+        echo "$package was not installed"
+        exit 1
+    fi
+done
--- a/images/linux/scripts/installers/python.sh
+++ b/images/linux/scripts/installers/python.sh
@@ -6,18 +6,40 @@

 set -e
 # Source the helpers for use with the script
+source $HELPER_SCRIPTS/etc-environment.sh
 source $HELPER_SCRIPTS/os.sh

 # Install Python, Python 3, pip, pip3
 if isUbuntu16 || isUbuntu18; then
-    apt-get install -y --no-install-recommends python python-dev python-pip python3 python3-dev python3-pip
+    apt-get install -y --no-install-recommends python python-dev python-pip python3 python3-dev python3-pip python3-venv
 fi

 if isUbuntu20; then
-    apt-get install -y --no-install-recommends python3 python3-dev python3-pip
+    apt-get install -y --no-install-recommends python3 python3-dev python3-pip python3-venv
    ln -s /usr/bin/pip3 /usr/bin/pip
 fi

+if isUbuntu18 || isUbuntu20 ; then
+    # Install pipx
+    # Set pipx custom directory
+    export PIPX_BIN_DIR=/opt/pipx_bin
+    export PIPX_HOME=/opt/pipx
+
+    python3 -m pip install pipx
+    python3 -m pipx ensurepath
+
+    # Update /etc/environment
+    setEtcEnvironmentVariable "PIPX_BIN_DIR" $PIPX_BIN_DIR
+    setEtcEnvironmentVariable "PIPX_HOME" $PIPX_HOME
+    prependEtcEnvironmentPath $PIPX_BIN_DIR
+
+    # Test pipx
+    if ! command -v pipx; then
+        echo "pipx was not installed or not found on PATH"
+        exit 1
+    fi
+fi
+
 # Run tests to determine that the software installed as expected
 echo "Testing to make sure that script performed as expected, and basic scenarios work"
 for cmd in python pip python3 pip3; do
--- a/images/linux/toolsets/toolset-1804.json
+++ b/images/linux/toolsets/toolset-1804.json
@@ -164,7 +164,6 @@
            "time",
            "unzip",
            "wget",
-            "yamllint",
            "zip"
        ]
    },
@@ -186,5 +185,15 @@
            "node:12-alpine",
            "ubuntu:14.04"
        ]
-    }
+    },
+    "pipx": [
+        {
+            "package": "yamllint",
+            "cmd": "yamllint"
+        },
+        {
+            "package": "aws-sam-cli",
+            "cmd": "sam"
+        }
+    ]
 }
--- a/images/linux/toolsets/toolset-2004.json
+++ b/images/linux/toolsets/toolset-2004.json
@@ -127,7 +127,6 @@
            "time",
            "unzip",
            "wget",
-            "yamllint",
            "zip"
        ]
    },
@@ -149,5 +148,15 @@
            "node:12-alpine",
            "ubuntu:14.04"
        ]
-    }
+    },
+    "pipx": [
+        {
+            "package": "yamllint",
+            "cmd": "yamllint"
+        },
+        {
+            "package": "aws-sam-cli",
+            "cmd": "sam"
+        }
+    ]
 }
--- a/images/linux/ubuntu1804.json
+++ b/images/linux/ubuntu1804.json
@@ -238,10 +238,10 @@
        {
            "type": "shell",
            "scripts": [
-                "{{template_dir}}/scripts/installers/aws-sam-cli.sh"
+                "{{template_dir}}/scripts/installers/pipx-packages.sh"
            ],
            "environment_vars": [
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}"
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}"
            ],
            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
        },
--- a/images/linux/ubuntu2004.json
+++ b/images/linux/ubuntu2004.json
@@ -240,10 +240,10 @@
        {
            "type": "shell",
            "scripts": [
-                "{{template_dir}}/scripts/installers/aws-sam-cli.sh"
+                "{{template_dir}}/scripts/installers/pipx-packages.sh"
            ],
            "environment_vars": [
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}"
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}"
            ],
            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
        },
--- a/images/macos/provision/configuration/configure-machine.sh
+++ b/images/macos/provision/configuration/configure-machine.sh
@@ -27,7 +27,4 @@ sudo "/Library/Application Support/VMware Tools/vmware-resolutionSet" 1176 885
 # sudo security delete-certificate -Z FF6797793A3CD798DC5B2ABEF56F73EDC9F83A64 /Library/Keychains/System.keychain
 curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer --output $HOME/AppleWWDRCAG3.cer --silent
 sudo security add-trusted-cert -d -r unspecified -k /Library/Keychains/System.keychain $HOME/AppleWWDRCAG3.cer
-rm $HOME/AppleWWDRCAG3.cer
-
-# Disable spotlight indexing to prevent possible high CPU usage after startup
-sudo mdutil -ai off
+rm $HOME/AppleWWDRCAG3.cer
--- a/images/macos/tests/Xcode.Tests.ps1
+++ b/images/macos/tests/Xcode.Tests.ps1
@@ -56,7 +56,7 @@ Describe "Xcode" {
    }

    Context "XCODE_DEVELOPER_DIR" {
-        $stableXcodeVersions = $testCases | Where-Object { Test-XcodeStableRelease -Version $_ }
+        $stableXcodeVersions = $XCODE_VERSIONS | ForEach-Object { $_.Split("_")[0] } | Where-Object { Test-XcodeStableRelease -Version $_ }
        $majorXcodeVersions = $stableXcodeVersions | ForEach-Object { $_.Split(".")[0] } | Select-Object -Unique
        $testCases = $majorXcodeVersions | ForEach-Object {
            $majorXcodeVersion = $_
--- a/images/macos/toolsets/toolset-10.14.json
+++ b/images/macos/toolsets/toolset-10.14.json
@@ -220,7 +220,8 @@
                "3.5.*",
                "3.6.*",
                "3.7.*",
-                "3.8.*"
+                "3.8.*",
+                "3.9.*"
            ]
        },
        {
--- a/images/macos/toolsets/toolset-10.15.json
+++ b/images/macos/toolsets/toolset-10.15.json
@@ -129,7 +129,8 @@
                "3.5.*",
                "3.6.*",
                "3.7.*",
-                "3.8.*"
+                "3.8.*",
+                "3.9.*"
            ]
        },
        {
--- a/images/macos/toolsets/toolset-11.0.json
+++ b/images/macos/toolsets/toolset-11.0.json
@@ -68,7 +68,8 @@
            "platform" : "darwin",
            "versions": [
                "3.7.*",
-                "3.8.*"
+                "3.8.*",
+                "3.9.*"
            ]
        },
        {