From 5d96d50f595c66e45212350b93ab77380d7b58fa Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Wed, 7 Oct 2020 09:29:37 +0300 Subject: [PATCH 1/7] added pipx and yamllint --- images/macos/provision/core/commonutils.sh | 1 - images/macos/provision/core/pipx-packages.sh | 25 +++++++++++++++++++ images/macos/provision/core/python.sh | 14 ++++++++++- .../SoftwareReport.Common.psm1 | 4 +++ .../SoftwareReport.Generator.ps1 | 2 ++ images/macos/templates/macOS-10.14.json | 7 ++++++ images/macos/templates/macOS-10.15.json | 7 ++++++ images/macos/templates/macOS-11.0.json | 7 ++++++ images/macos/toolsets/toolset-10.14.json | 6 +++++ images/macos/toolsets/toolset-10.15.json | 14 ++++++++--- images/macos/toolsets/toolset-11.0.json | 10 ++++++-- 11 files changed, 89 insertions(+), 8 deletions(-) create mode 100644 images/macos/provision/core/pipx-packages.sh diff --git a/images/macos/provision/core/commonutils.sh b/images/macos/provision/core/commonutils.sh index 9b32d5c2..5b3522f5 100644 --- a/images/macos/provision/core/commonutils.sh +++ b/images/macos/provision/core/commonutils.sh @@ -22,7 +22,6 @@ binst_common_utils=( github/gh/gh p7zip ant - yamllint aria2 ) diff --git a/images/macos/provision/core/pipx-packages.sh b/images/macos/provision/core/pipx-packages.sh new file mode 100644 index 00000000..c01a4b97 --- /dev/null +++ b/images/macos/provision/core/pipx-packages.sh @@ -0,0 +1,25 @@ +source ~/utils/utils.sh + +export PATH="$PATH:/opt/pipx_bin" + +toolset=get_toolset_path +pipx_packages=$(jq -r ".pipx[] .package" $toolset) + +for package in $pipx_packages; do + python_version=$(jq -r ".pipx[] | select(.package == \"$package\") .python" $toolset) + if [ "$python_version" != "null" ]; then + python_path="/opt/hostedtoolcache/Python/$python_version*/x64/bin/python$python_version" + echo "Install $package into python $python_path" + pipx install $package --python $python_path + else + echo "Install $package into default python" + pipx install $package + fi + + # Run tests to determine that the software installed as expected + cmd=$(jq -r ".pipx[] | select(.package == \"$package\") .cmd" $toolset) + if ! command -v $cmd; then + echo "$package was not installed" + exit 1 + fi +done \ No newline at end of file diff --git a/images/macos/provision/core/python.sh b/images/macos/provision/core/python.sh index 41488642..5bda9ef3 100755 --- a/images/macos/provision/core/python.sh +++ b/images/macos/provision/core/python.sh @@ -7,7 +7,19 @@ echo "Brew Installing Python 3" /usr/local/bin/brew install python3 echo "Brew Installing Python 2" -# Create local tap with formula due to python2 formula depreciation +# Create local tap with formula due to python2 formula depreciation /usr/local/bin/brew tap-new local/python2 FORMULA_PATH=$(/usr/local/bin/brew extract python@2 local/python2 | grep "Homebrew/Library/Taps") /usr/local/bin/brew install $FORMULA_PATH + +echo "Installing pipx" +export PIPX_BIN_DIR=/opt/pipx_bin +export PIPX_HOME=/opt/pipx + +brew install pipx +python3 -m pipx ensurepath + +echo "export PIPX_BIN_DIR=${PIPX_BIN_DIR}" >> "${HOME}/.bashrc" +echo "export PIPX_HOME=${PIPX_HOME}" >> "${HOME}/.bashrc" +echo 'export PATH="$PIPX_BIN_DIR:$PATH"' >> "${HOME}/.bashrc" + diff --git a/images/macos/software-report/SoftwareReport.Common.psm1 b/images/macos/software-report/SoftwareReport.Common.psm1 index c3fe7517..5c518d16 100644 --- a/images/macos/software-report/SoftwareReport.Common.psm1 +++ b/images/macos/software-report/SoftwareReport.Common.psm1 @@ -96,6 +96,10 @@ function Get-PipVersion { return "${versionPart1} ${versionPart2} ${versionPart3}" } +function Get-PipxVersion { + return "Pipx $(pipx --version 2> $null)" +} + function Get-NVMNodeVersionList { $nvmPath = Join-Path $env:HOME ".nvm" "nvm.sh" $nvmInitCommand = ". ${nvmPath} > /dev/null 2>&1 || true" diff --git a/images/macos/software-report/SoftwareReport.Generator.ps1 b/images/macos/software-report/SoftwareReport.Generator.ps1 index 87c94459..40214d7a 100644 --- a/images/macos/software-report/SoftwareReport.Generator.ps1 +++ b/images/macos/software-report/SoftwareReport.Generator.ps1 @@ -81,6 +81,7 @@ $npmVersion = Run-Command "npm --version" $yarnVersion = Run-Command "yarn --version" $nugetVersion = Run-Command "nuget help" | Select-Object -First 1 | Take-Part -Part 2 $pip3Version = Get-PipVersion -Version 3 +$pipxVersion = Get-PipxVersion $condaVersion = Invoke-Expression "conda --version" $rubyGemsVersion = Run-Command "gem --version" $composerVersion = Run-Command "composer --version" | Take-Part -Part 2 @@ -97,6 +98,7 @@ if ($os.IsLessThanBigSur) { $markdown += New-MDList -Style Unordered -Lines @( "Pip ${pip3Version}", + $pipxVersion, $bundlerVersion, "Carthage ${carthageVersion}", "CocoaPods ${cocoaPodsVersion}", diff --git a/images/macos/templates/macOS-10.14.json b/images/macos/templates/macOS-10.14.json index 269ffa71..3cb0b070 100644 --- a/images/macos/templates/macOS-10.14.json +++ b/images/macos/templates/macOS-10.14.json @@ -196,6 +196,13 @@ "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} pwsh -f {{ .Path }}", "scripts": "./provision/core/toolset.ps1" }, + { + "type": "shell", + "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}", + "scripts": [ + "./provision/core/pipx-packages.sh" + ] + }, { "type": "shell", "execute_command": "ruby {{ .Path }}", diff --git a/images/macos/templates/macOS-10.15.json b/images/macos/templates/macOS-10.15.json index 3dcc3d33..65a0d12d 100644 --- a/images/macos/templates/macOS-10.15.json +++ b/images/macos/templates/macOS-10.15.json @@ -195,6 +195,13 @@ "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} pwsh -f {{ .Path }}", "scripts": "./provision/core/toolset.ps1" }, + { + "type": "shell", + "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}", + "scripts": [ + "./provision/core/pipx-packages.sh" + ] + }, { "type": "shell", "execute_command": "ruby {{ .Path }}", diff --git a/images/macos/templates/macOS-11.0.json b/images/macos/templates/macOS-11.0.json index 6da9df38..c732f799 100644 --- a/images/macos/templates/macOS-11.0.json +++ b/images/macos/templates/macOS-11.0.json @@ -192,6 +192,13 @@ "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} pwsh -f {{ .Path }}", "scripts": "./provision/core/toolset.ps1" }, + { + "type": "shell", + "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}", + "scripts": [ + "./provision/core/pipx-packages.sh" + ] + }, { "type": "shell", "execute_command": "ruby {{ .Path }}", diff --git a/images/macos/toolsets/toolset-10.14.json b/images/macos/toolsets/toolset-10.14.json index e868f769..fc1d44af 100644 --- a/images/macos/toolsets/toolset-10.14.json +++ b/images/macos/toolsets/toolset-10.14.json @@ -257,5 +257,11 @@ "1.15.*" ] } + ], + "pipx": [ + { + "package": "yamllint", + "cmd": "yamllint" + } ] } \ No newline at end of file diff --git a/images/macos/toolsets/toolset-10.15.json b/images/macos/toolsets/toolset-10.15.json index 8497a710..e754bacf 100644 --- a/images/macos/toolsets/toolset-10.15.json +++ b/images/macos/toolsets/toolset-10.15.json @@ -26,28 +26,28 @@ "mono":"6.12", "ios": "14.0", "mac": "6.20", - "android": "11.0" + "android": "11.0" }, { "symlink": "6_12_0", "mono":"6.12", "ios": "13.20", "mac": "6.20", - "android": "11.0" + "android": "11.0" }, { "symlink": "6_10_0", "mono":"6.10", "ios": "13.18", "mac": "6.18", - "android": "10.3" + "android": "10.3" }, { "symlink": "6_8_1", "mono":"6.8", "ios": "13.16", "mac": "6.16", - "android": "10.2" + "android": "10.2" }, { "symlink": "6_8_0", @@ -166,5 +166,11 @@ "1.15.*" ] } + ], + "pipx": [ + { + "package": "yamllint", + "cmd": "yamllint" + } ] } \ No newline at end of file diff --git a/images/macos/toolsets/toolset-11.0.json b/images/macos/toolsets/toolset-11.0.json index 725da218..8a739f91 100644 --- a/images/macos/toolsets/toolset-11.0.json +++ b/images/macos/toolsets/toolset-11.0.json @@ -26,14 +26,14 @@ "mono":"6.12", "ios": "14.0", "mac": "6.20", - "android": "11.0" + "android": "11.0" }, { "symlink": "6_12_0", "mono":"6.12", "ios": "13.20", "mac": "6.20", - "android": "11.0" + "android": "11.0" } ] }, @@ -91,5 +91,11 @@ "1.15.*" ] } + ], + "pipx": [ + { + "package": "yamllint", + "cmd": "yamllint" + } ] } \ No newline at end of file From 6742dd7a546df5c182a04a552d62ebb6eaaa41b9 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Wed, 7 Oct 2020 14:58:13 +0300 Subject: [PATCH 2/7] resolved comments --- images/macos/provision/core/pipx-packages.sh | 9 +-------- images/macos/provision/core/python.sh | 5 ++--- images/macos/tests/Python.Tests.ps1 | 4 ++++ 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/images/macos/provision/core/pipx-packages.sh b/images/macos/provision/core/pipx-packages.sh index c01a4b97..b755dede 100644 --- a/images/macos/provision/core/pipx-packages.sh +++ b/images/macos/provision/core/pipx-packages.sh @@ -8,18 +8,11 @@ pipx_packages=$(jq -r ".pipx[] .package" $toolset) for package in $pipx_packages; do python_version=$(jq -r ".pipx[] | select(.package == \"$package\") .python" $toolset) if [ "$python_version" != "null" ]; then - python_path="/opt/hostedtoolcache/Python/$python_version*/x64/bin/python$python_version" + python_path="$HOME/hostedtoolcache/Python/$python_version*/x64/bin/python$python_version" echo "Install $package into python $python_path" pipx install $package --python $python_path else echo "Install $package into default python" pipx install $package fi - - # Run tests to determine that the software installed as expected - cmd=$(jq -r ".pipx[] | select(.package == \"$package\") .cmd" $toolset) - if ! command -v $cmd; then - echo "$package was not installed" - exit 1 - fi done \ No newline at end of file diff --git a/images/macos/provision/core/python.sh b/images/macos/provision/core/python.sh index 5bda9ef3..0c4058e3 100755 --- a/images/macos/provision/core/python.sh +++ b/images/macos/provision/core/python.sh @@ -13,11 +13,10 @@ FORMULA_PATH=$(/usr/local/bin/brew extract python@2 local/python2 | grep "Homebr /usr/local/bin/brew install $FORMULA_PATH echo "Installing pipx" -export PIPX_BIN_DIR=/opt/pipx_bin -export PIPX_HOME=/opt/pipx +export PIPX_BIN_DIR=/usr/local/opt/pipx_bin +export PIPX_HOME=/usr/local/opt/pipx brew install pipx -python3 -m pipx ensurepath echo "export PIPX_BIN_DIR=${PIPX_BIN_DIR}" >> "${HOME}/.bashrc" echo "export PIPX_HOME=${PIPX_HOME}" >> "${HOME}/.bashrc" diff --git a/images/macos/tests/Python.Tests.ps1 b/images/macos/tests/Python.Tests.ps1 index 76cd8941..43dc122a 100644 --- a/images/macos/tests/Python.Tests.ps1 +++ b/images/macos/tests/Python.Tests.ps1 @@ -31,4 +31,8 @@ Describe "Python" { It "Pip 3 is available" { "pip3 --version" | Should -ReturnZeroExitCode } + + It "Pipx is available" { + "pipx --version" | Should -ReturnZeroExitCode + } } \ No newline at end of file From ef1a055d44d3210760ee96ac96ec0c124e72cd68 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Wed, 7 Oct 2020 16:20:27 +0300 Subject: [PATCH 3/7] added pipx to 10.13 --- images/macos/templates/macOS-10.13.json | 7 +++++++ images/macos/toolsets/toolset-10.13.json | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/images/macos/templates/macOS-10.13.json b/images/macos/templates/macOS-10.13.json index 2190a6b4..2300bf51 100644 --- a/images/macos/templates/macOS-10.13.json +++ b/images/macos/templates/macOS-10.13.json @@ -183,6 +183,13 @@ "./provision/core/pypy.sh" ] }, + { + "type": "shell", + "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}", + "scripts": [ + "./provision/core/pipx-packages.sh" + ] + }, { "type": "shell", "inline": [ diff --git a/images/macos/toolsets/toolset-10.13.json b/images/macos/toolsets/toolset-10.13.json index fdb14d08..5157fe47 100644 --- a/images/macos/toolsets/toolset-10.13.json +++ b/images/macos/toolsets/toolset-10.13.json @@ -198,5 +198,11 @@ "3.6" ] } + ], + "pipx": [ + { + "package": "yamllint", + "cmd": "yamllint" + } ] } \ No newline at end of file From 451598b8c171adf7fea6da3715e7e41cdee0d368 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Fri, 9 Oct 2020 09:43:44 +0300 Subject: [PATCH 4/7] fixed pipx-packages.sh --- images/macos/provision/core/pipx-packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/macos/provision/core/pipx-packages.sh b/images/macos/provision/core/pipx-packages.sh index b755dede..658600fa 100644 --- a/images/macos/provision/core/pipx-packages.sh +++ b/images/macos/provision/core/pipx-packages.sh @@ -2,7 +2,7 @@ source ~/utils/utils.sh export PATH="$PATH:/opt/pipx_bin" -toolset=get_toolset_path +toolset=$(get_toolset_path) pipx_packages=$(jq -r ".pipx[] .package" $toolset) for package in $pipx_packages; do From f1da709f47f9f14aa559501fd260f2dcdc1448de Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Mon, 12 Oct 2020 11:10:02 +0300 Subject: [PATCH 5/7] fixed Get-PipxVersion --- images/macos/software-report/SoftwareReport.Common.psm1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/images/macos/software-report/SoftwareReport.Common.psm1 b/images/macos/software-report/SoftwareReport.Common.psm1 index 5c518d16..7616b81b 100644 --- a/images/macos/software-report/SoftwareReport.Common.psm1 +++ b/images/macos/software-report/SoftwareReport.Common.psm1 @@ -97,7 +97,8 @@ function Get-PipVersion { } function Get-PipxVersion { - return "Pipx $(pipx --version 2> $null)" + $pipxVersion = Run-Command "pipx --version" -SuppressStderr + return "Pipx $pipxVersion" } function Get-NVMNodeVersionList { From edff9f791bade856d31e67e25fa191500d429c96 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Wed, 21 Oct 2020 08:33:58 +0300 Subject: [PATCH 6/7] updated json files --- images/macos/templates/macOS-10.13.json | 8 +------- images/macos/templates/macOS-10.14.json | 10 ++-------- images/macos/templates/macOS-10.15.json | 10 ++-------- images/macos/templates/macOS-11.0.json | 10 ++-------- 4 files changed, 7 insertions(+), 31 deletions(-) diff --git a/images/macos/templates/macOS-10.13.json b/images/macos/templates/macOS-10.13.json index aa43650a..9ebd56e6 100644 --- a/images/macos/templates/macOS-10.13.json +++ b/images/macos/templates/macOS-10.13.json @@ -181,13 +181,7 @@ "./provision/core/edge.sh", "./provision/core/firefox.sh", "./provision/core/toolcache-high-sierra.sh", - "./provision/core/pypy.sh" - ] - }, - { - "type": "shell", - "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}", - "scripts": [ + "./provision/core/pypy.sh", "./provision/core/pipx-packages.sh" ] }, diff --git a/images/macos/templates/macOS-10.14.json b/images/macos/templates/macOS-10.14.json index 017bc607..c03cc72a 100644 --- a/images/macos/templates/macOS-10.14.json +++ b/images/macos/templates/macOS-10.14.json @@ -186,7 +186,8 @@ "./provision/core/miniconda.sh", "./provision/core/xcode-postbuild.sh", "./provision/core/toolcache.sh", - "./provision/core/pypy.sh" + "./provision/core/pypy.sh", + "./provision/core/pipx-packages.sh" ], "environment_vars": [ "GITHUB_FEED_TOKEN={{user `github_feed_token`}}" @@ -197,13 +198,6 @@ "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} pwsh -f {{ .Path }}", "scripts": "./provision/core/toolset.ps1" }, - { - "type": "shell", - "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}", - "scripts": [ - "./provision/core/pipx-packages.sh" - ] - }, { "type": "shell", "execute_command": "ruby {{ .Path }}", diff --git a/images/macos/templates/macOS-10.15.json b/images/macos/templates/macOS-10.15.json index cf225d41..cf49ef57 100644 --- a/images/macos/templates/macOS-10.15.json +++ b/images/macos/templates/macOS-10.15.json @@ -185,7 +185,8 @@ "./provision/core/firefox.sh", "./provision/core/xcode-postbuild.sh", "./provision/core/toolcache.sh", - "./provision/core/pypy.sh" + "./provision/core/pypy.sh", + "./provision/core/pipx-packages.sh" ], "environment_vars": [ "GITHUB_FEED_TOKEN={{user `github_feed_token`}}" @@ -196,13 +197,6 @@ "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} pwsh -f {{ .Path }}", "scripts": "./provision/core/toolset.ps1" }, - { - "type": "shell", - "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}", - "scripts": [ - "./provision/core/pipx-packages.sh" - ] - }, { "type": "shell", "execute_command": "ruby {{ .Path }}", diff --git a/images/macos/templates/macOS-11.0.json b/images/macos/templates/macOS-11.0.json index d1f54d31..585a9354 100644 --- a/images/macos/templates/macOS-11.0.json +++ b/images/macos/templates/macOS-11.0.json @@ -181,7 +181,8 @@ "./provision/core/chrome.sh", "./provision/core/edge.sh", "./provision/core/firefox.sh", - "./provision/core/toolcache.sh" + "./provision/core/toolcache.sh", + "./provision/core/pipx-packages.sh" ], "environment_vars": [ "GITHUB_FEED_TOKEN={{user `github_feed_token`}}" @@ -192,13 +193,6 @@ "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} pwsh -f {{ .Path }}", "scripts": "./provision/core/toolset.ps1" }, - { - "type": "shell", - "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} {{ .Path }}", - "scripts": [ - "./provision/core/pipx-packages.sh" - ] - }, { "type": "shell", "execute_command": "ruby {{ .Path }}", From 00bb4818cec8f8676369d59f0eb33f7802ca37f2 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Wed, 21 Oct 2020 15:50:53 +0300 Subject: [PATCH 7/7] fixed pipx packages tests --- images/macos/tests/Linters.Tests.ps1 | 4 ---- images/macos/tests/PipxPackages.Tests.ps1 | 7 +++++++ images/macos/toolsets/toolset-10.13.json | 2 +- images/macos/toolsets/toolset-10.14.json | 2 +- images/macos/toolsets/toolset-10.15.json | 2 +- images/macos/toolsets/toolset-11.0.json | 2 +- 6 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 images/macos/tests/PipxPackages.Tests.ps1 diff --git a/images/macos/tests/Linters.Tests.ps1 b/images/macos/tests/Linters.Tests.ps1 index 3b7b8e0d..31d3698c 100644 --- a/images/macos/tests/Linters.Tests.ps1 +++ b/images/macos/tests/Linters.Tests.ps1 @@ -3,10 +3,6 @@ Import-Module "$PSScriptRoot/../helpers/Tests.Helpers.psm1" $os = Get-OSVersion Describe "Linters" { - It "yamllint" { - "yamllint --version" | Should -ReturnZeroExitCode - } - It "SwiftLint" -Skip:($os.IsHighSierra) { "swiftlint version" | Should -ReturnZeroExitCode } diff --git a/images/macos/tests/PipxPackages.Tests.ps1 b/images/macos/tests/PipxPackages.Tests.ps1 new file mode 100644 index 00000000..ed981938 --- /dev/null +++ b/images/macos/tests/PipxPackages.Tests.ps1 @@ -0,0 +1,7 @@ +Describe "PipxPackages" { + $pipxToolset = Get-ToolsetValue "pipx" + $testCases = $pipxToolset | ForEach-Object { @{package = $_.package; cmd = $_.cmd} } + It "" -TestCases $testCases { + "$cmd" | Should -ReturnZeroExitCode + } +} \ No newline at end of file diff --git a/images/macos/toolsets/toolset-10.13.json b/images/macos/toolsets/toolset-10.13.json index f1ce3b06..3bf62968 100644 --- a/images/macos/toolsets/toolset-10.13.json +++ b/images/macos/toolsets/toolset-10.13.json @@ -198,7 +198,7 @@ "pipx": [ { "package": "yamllint", - "cmd": "yamllint" + "cmd": "yamllint --version" } ] } \ No newline at end of file diff --git a/images/macos/toolsets/toolset-10.14.json b/images/macos/toolsets/toolset-10.14.json index f1c70a83..aa1b4101 100644 --- a/images/macos/toolsets/toolset-10.14.json +++ b/images/macos/toolsets/toolset-10.14.json @@ -258,7 +258,7 @@ "pipx": [ { "package": "yamllint", - "cmd": "yamllint" + "cmd": "yamllint --version" } ] } \ No newline at end of file diff --git a/images/macos/toolsets/toolset-10.15.json b/images/macos/toolsets/toolset-10.15.json index e2debc9f..c54db919 100644 --- a/images/macos/toolsets/toolset-10.15.json +++ b/images/macos/toolsets/toolset-10.15.json @@ -167,7 +167,7 @@ "pipx": [ { "package": "yamllint", - "cmd": "yamllint" + "cmd": "yamllint --version" } ] } \ No newline at end of file diff --git a/images/macos/toolsets/toolset-11.0.json b/images/macos/toolsets/toolset-11.0.json index e5cf3e64..ed8903d3 100644 --- a/images/macos/toolsets/toolset-11.0.json +++ b/images/macos/toolsets/toolset-11.0.json @@ -92,7 +92,7 @@ "pipx": [ { "package": "yamllint", - "cmd": "yamllint" + "cmd": "yamllint --version" } ] } \ No newline at end of file