From 48c3a310a5606b9505f8f4d0ca76927d0d997505 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Thu, 8 Oct 2020 16:57:24 +0300 Subject: [PATCH 01/23] added pipx and yamllint --- images/win/Windows2016-Azure.json | 8 +++++++- images/win/Windows2019-Azure.json | 8 +++++++- .../scripts/Installers/Install-Pipx-Packages.ps1 | 12 ++++++++++++ images/win/scripts/Installers/Install-Pipx.ps1 | 14 ++++++++++++++ images/win/scripts/Installers/Install-YAMLLint.ps1 | 3 --- .../SoftwareReport/SoftwareReport.Common.psm1 | 4 ++++ .../SoftwareReport/SoftwareReport.Generator.ps1 | 3 ++- images/win/scripts/Tests/PipxPackages.Test.ps1 | 5 +++++ images/win/scripts/Tests/Tools.Tests.ps1 | 12 ++++++------ images/win/toolsets/toolset-2016.json | 8 +++++++- images/win/toolsets/toolset-2019.json | 8 +++++++- 11 files changed, 71 insertions(+), 14 deletions(-) create mode 100644 images/win/scripts/Installers/Install-Pipx-Packages.ps1 create mode 100644 images/win/scripts/Installers/Install-Pipx.ps1 delete mode 100644 images/win/scripts/Installers/Install-YAMLLint.ps1 create mode 100644 images/win/scripts/Tests/PipxPackages.Test.ps1 diff --git a/images/win/Windows2016-Azure.json b/images/win/Windows2016-Azure.json index d65286497..f2389d423 100644 --- a/images/win/Windows2016-Azure.json +++ b/images/win/Windows2016-Azure.json @@ -235,7 +235,6 @@ "{{ template_dir }}/scripts/Installers/Install-PyPy.ps1", "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1", "{{ template_dir }}/scripts/Installers/Configure-Toolset.ps1", - "{{ template_dir }}/scripts/Installers/Install-YAMLLint.ps1", "{{ template_dir }}/scripts/Installers/Update-AndroidSDK.ps1" ] }, @@ -328,6 +327,13 @@ "{{ template_dir }}/scripts/Installers/Install-CodeQLBundle.ps1" ] }, + { + "type": "powershell", + "scripts": [ + "{{ template_dir }}/scripts/Installers/Install-Pipx.ps1", + "{{ template_dir }}/scripts/Installers/Install-Pipx-Packages.ps1" + ] + }, { "type": "powershell", "scripts": [ diff --git a/images/win/Windows2019-Azure.json b/images/win/Windows2019-Azure.json index 6e0ba8669..7e082f20e 100644 --- a/images/win/Windows2019-Azure.json +++ b/images/win/Windows2019-Azure.json @@ -249,7 +249,6 @@ "{{ template_dir }}/scripts/Installers/Install-PyPy.ps1", "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1", "{{ template_dir }}/scripts/Installers/Configure-Toolset.ps1", - "{{ template_dir }}/scripts/Installers/Install-YAMLLint.ps1", "{{ template_dir }}/scripts/Installers/Update-AndroidSDK.ps1", "{{ template_dir }}/scripts/Installers/Install-AzureModules.ps1" ] @@ -326,6 +325,13 @@ "{{ template_dir }}/scripts/Installers/Install-CodeQLBundle.ps1" ] }, + { + "type": "powershell", + "scripts": [ + "{{ template_dir }}/scripts/Installers/Install-Pipx.ps1", + "{{ template_dir }}/scripts/Installers/Install-Pipx-Packages.ps1" + ] + }, { "type": "powershell", "scripts": [ diff --git a/images/win/scripts/Installers/Install-Pipx-Packages.ps1 b/images/win/scripts/Installers/Install-Pipx-Packages.ps1 new file mode 100644 index 000000000..a5be7b6ff --- /dev/null +++ b/images/win/scripts/Installers/Install-Pipx-Packages.ps1 @@ -0,0 +1,12 @@ +$pipxToolset = (Get-ToolsetContent).pipx +foreach($tool in $pipxToolset) { + if ($tool.python) { + $pythonPath = "${env:AGENT_TOOLSDIRECTORY}\Python\${tool.python}*\${env:AGENT_OSARCHITECTURE}\python-${tool.python}*" + Write-Host "Install ${tool.package} into python ${tool.python}" + pipx install $tool.package --python $python_path + } else { + Write-Host "Install ${tool.package} into default python" + pipx install $tool.package + } + Invoke-PesterTests -TestFile "PipxPackages" -TestName $tool.package +} \ No newline at end of file diff --git a/images/win/scripts/Installers/Install-Pipx.ps1 b/images/win/scripts/Installers/Install-Pipx.ps1 new file mode 100644 index 000000000..a78cf456b --- /dev/null +++ b/images/win/scripts/Installers/Install-Pipx.ps1 @@ -0,0 +1,14 @@ +# Set custom directorys for pipx +$env:PIPX_BIN_DIR = "${env:ProgramFiles(x86)}\pipx_bin" +$env:PIPX_HOME = "${env:ProgramFiles(x86)}\pipx" + +# Install pipx +pip install pipx + +# Set environment variables +Add-MachinePathItem "${env:PIPX_BIN_DIR}" +Set-SystemVariable -SystemVariable PIPX_BIN_DIR -Value $env:PIPX_BIN_DIR +Set-SystemVariable -SystemVariable PIPX_HOME -Value $env:PIPX_HOME + +# Test pipx +Invoke-PesterTests -TestFile "Tools" -TestName "Pipx" \ No newline at end of file diff --git a/images/win/scripts/Installers/Install-YAMLLint.ps1 b/images/win/scripts/Installers/Install-YAMLLint.ps1 deleted file mode 100644 index 1ebc09a33..000000000 --- a/images/win/scripts/Installers/Install-YAMLLint.ps1 +++ /dev/null @@ -1,3 +0,0 @@ -pip install yamllint - -Invoke-PesterTests -TestFile "Tools" -TestName "yamllint" \ No newline at end of file diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 b/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 index 05b101aac..12f41061c 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 @@ -273,4 +273,8 @@ function Get-PacmanVersion { function Get-YAMLLintVersion { yamllint --version +} + +function Get-PipxVersion { + pipx --version } \ No newline at end of file diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 index f83157748..ef7749e31 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 @@ -59,7 +59,8 @@ $markdown += New-MDList -Style Unordered -Lines @( (Get-RubyGemsVersion), (Get-HelmVersion), (Get-ComposerVersion), - (Get-NugetVersion) + (Get-NugetVersion), + (Get-PipxVersion) ) $markdown += New-MDHeader "Project Management" -Level 3 diff --git a/images/win/scripts/Tests/PipxPackages.Test.ps1 b/images/win/scripts/Tests/PipxPackages.Test.ps1 new file mode 100644 index 000000000..3a9ad5644 --- /dev/null +++ b/images/win/scripts/Tests/PipxPackages.Test.ps1 @@ -0,0 +1,5 @@ +Describe "yamllint" { + It "yamllint" { + "yamllint --version" | Should -ReturnZeroExitCode + } +} \ No newline at end of file diff --git a/images/win/scripts/Tests/Tools.Tests.ps1 b/images/win/scripts/Tests/Tools.Tests.ps1 index cbb0609a5..a00245321 100644 --- a/images/win/scripts/Tests/Tools.Tests.ps1 +++ b/images/win/scripts/Tests/Tools.Tests.ps1 @@ -214,12 +214,6 @@ Describe "ServiceFabricSDK" { } } -Describe "yamllint" { - It "yamllint" { - "yamllint --version" | Should -ReturnZeroExitCode - } -} - Describe "Svn" { It "svn" { "svn --version --quiet" | Should -ReturnZeroExitCode @@ -269,3 +263,9 @@ Describe "Zstd" { "zstd -V" | Should -ReturnZeroExitCode } } + +Describe "Pipx" { + It "Pipx" { + "pipx --version" | Should -ReturnZeroExitCode + } +} diff --git a/images/win/toolsets/toolset-2016.json b/images/win/toolsets/toolset-2016.json index 5e0b23c6b..441b34c18 100644 --- a/images/win/toolsets/toolset-2016.json +++ b/images/win/toolsets/toolset-2016.json @@ -236,5 +236,11 @@ "mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2016", "microsoft/aspnetcore-build:1.0-2.0" ] - } + }, + "pipx": [ + { + "package": "yamllint", + "cmd": "yamllint" + } + ] } diff --git a/images/win/toolsets/toolset-2019.json b/images/win/toolsets/toolset-2019.json index bf1d0dff6..275e63e58 100644 --- a/images/win/toolsets/toolset-2019.json +++ b/images/win/toolsets/toolset-2019.json @@ -288,5 +288,11 @@ "mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2019", "microsoft/aspnetcore-build:1.0-2.0" ] - } + }, + "pipx": [ + { + "package": "yamllint", + "cmd": "yamllint" + } + ] } From f683996383d8de2298e6f62ed79da331f1034d86 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Fri, 9 Oct 2020 09:37:58 +0300 Subject: [PATCH 02/23] fixed Install-PipxPackages.ps1 and tests --- ...nstall-Pipx-Packages.ps1 => Install-PipxPackages.ps1} | 8 ++++---- images/win/scripts/Tests/PipxPackages.Test.ps1 | 9 ++++++--- images/win/toolsets/toolset-2019.json | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) rename images/win/scripts/Installers/{Install-Pipx-Packages.ps1 => Install-PipxPackages.ps1} (51%) diff --git a/images/win/scripts/Installers/Install-Pipx-Packages.ps1 b/images/win/scripts/Installers/Install-PipxPackages.ps1 similarity index 51% rename from images/win/scripts/Installers/Install-Pipx-Packages.ps1 rename to images/win/scripts/Installers/Install-PipxPackages.ps1 index a5be7b6ff..e83f5add4 100644 --- a/images/win/scripts/Installers/Install-Pipx-Packages.ps1 +++ b/images/win/scripts/Installers/Install-PipxPackages.ps1 @@ -1,12 +1,12 @@ $pipxToolset = (Get-ToolsetContent).pipx foreach($tool in $pipxToolset) { if ($tool.python) { - $pythonPath = "${env:AGENT_TOOLSDIRECTORY}\Python\${tool.python}*\${env:AGENT_OSARCHITECTURE}\python-${tool.python}*" + $pythonPath = (Get-Item -Path "${env:AGENT_TOOLSDIRECTORY}\Python\${tool.python}.*\x64\python-${tool.python}*").FullName Write-Host "Install ${tool.package} into python ${tool.python}" - pipx install $tool.package --python $python_path + pipx install $tool.package --python $pythonPath } else { Write-Host "Install ${tool.package} into default python" pipx install $tool.package } - Invoke-PesterTests -TestFile "PipxPackages" -TestName $tool.package -} \ No newline at end of file +} +Invoke-PesterTests -TestFile "PipxPackages" \ No newline at end of file diff --git a/images/win/scripts/Tests/PipxPackages.Test.ps1 b/images/win/scripts/Tests/PipxPackages.Test.ps1 index 3a9ad5644..fbb4c5964 100644 --- a/images/win/scripts/Tests/PipxPackages.Test.ps1 +++ b/images/win/scripts/Tests/PipxPackages.Test.ps1 @@ -1,5 +1,8 @@ -Describe "yamllint" { - It "yamllint" { - "yamllint --version" | Should -ReturnZeroExitCode +Describe "PipxPackages" { + $pipxToolset = (Get-ToolsetContent).pipx + foreach($tool in $pipxToolset) { + It "${tool.package}" { + "${tool.cmd}" | Should -ReturnZeroExitCode + } } } \ No newline at end of file diff --git a/images/win/toolsets/toolset-2019.json b/images/win/toolsets/toolset-2019.json index 275e63e58..5ca23b4ed 100644 --- a/images/win/toolsets/toolset-2019.json +++ b/images/win/toolsets/toolset-2019.json @@ -292,7 +292,7 @@ "pipx": [ { "package": "yamllint", - "cmd": "yamllint" + "cmd": "yamllint --version" } ] } From 08f5d6592b9a824d503397af2174f0b1f1d74879 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Fri, 9 Oct 2020 11:31:54 +0300 Subject: [PATCH 03/23] updated tests --- images/win/scripts/Tests/PipxPackages.Test.ps1 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/images/win/scripts/Tests/PipxPackages.Test.ps1 b/images/win/scripts/Tests/PipxPackages.Test.ps1 index fbb4c5964..b41b8f8f8 100644 --- a/images/win/scripts/Tests/PipxPackages.Test.ps1 +++ b/images/win/scripts/Tests/PipxPackages.Test.ps1 @@ -1,8 +1,7 @@ Describe "PipxPackages" { $pipxToolset = (Get-ToolsetContent).pipx - foreach($tool in $pipxToolset) { - It "${tool.package}" { - "${tool.cmd}" | Should -ReturnZeroExitCode - } + $testCases = $pipxToolset | ForEach-Object { @{package = $_.package; cmd = $_.cmd} } + It "" -TestCases $testCases { + "$cmd" | Should -ReturnZeroExitCode } } \ No newline at end of file From 2977e6c42cbe552b8b572a6d39663b2a2fbad92b Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Fri, 9 Oct 2020 11:33:25 +0300 Subject: [PATCH 04/23] fixed naming --- images/win/windows2016.json | 2 +- images/win/windows2019.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/images/win/windows2016.json b/images/win/windows2016.json index f2389d423..21bcfff44 100644 --- a/images/win/windows2016.json +++ b/images/win/windows2016.json @@ -331,7 +331,7 @@ "type": "powershell", "scripts": [ "{{ template_dir }}/scripts/Installers/Install-Pipx.ps1", - "{{ template_dir }}/scripts/Installers/Install-Pipx-Packages.ps1" + "{{ template_dir }}/scripts/Installers/Install-PipxPackages.ps1" ] }, { diff --git a/images/win/windows2019.json b/images/win/windows2019.json index 7e082f20e..62bf1b941 100644 --- a/images/win/windows2019.json +++ b/images/win/windows2019.json @@ -329,7 +329,7 @@ "type": "powershell", "scripts": [ "{{ template_dir }}/scripts/Installers/Install-Pipx.ps1", - "{{ template_dir }}/scripts/Installers/Install-Pipx-Packages.ps1" + "{{ template_dir }}/scripts/Installers/Install-PipxPackages.ps1" ] }, { From 42b6aa68b4e6241bb1e961aa002d92a3ab96c7d3 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Thu, 15 Oct 2020 12:02:07 +0300 Subject: [PATCH 05/23] added TOOLSET_JSON_PATH --- images/win/windows2016.json | 3 +++ images/win/windows2019.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/images/win/windows2016.json b/images/win/windows2016.json index 21bcfff44..8995569e2 100644 --- a/images/win/windows2016.json +++ b/images/win/windows2016.json @@ -329,6 +329,9 @@ }, { "type": "powershell", + "environment_vars": [ + "TOOLSET_JSON_PATH={{user `toolset_json_path`}}" + ], "scripts": [ "{{ template_dir }}/scripts/Installers/Install-Pipx.ps1", "{{ template_dir }}/scripts/Installers/Install-PipxPackages.ps1" diff --git a/images/win/windows2019.json b/images/win/windows2019.json index 62bf1b941..ced90aad1 100644 --- a/images/win/windows2019.json +++ b/images/win/windows2019.json @@ -327,6 +327,9 @@ }, { "type": "powershell", + "environment_vars": [ + "TOOLSET_JSON_PATH={{user `toolset_json_path`}}" + ], "scripts": [ "{{ template_dir }}/scripts/Installers/Install-Pipx.ps1", "{{ template_dir }}/scripts/Installers/Install-PipxPackages.ps1" From 723e4ccb877619a062dd718065122450708c33d1 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Fri, 16 Oct 2020 12:36:59 +0300 Subject: [PATCH 06/23] fixed naming --- ...{PipxPackages.Test.ps1 => PipxPackages.Tests.ps1} | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) rename images/win/scripts/Tests/{PipxPackages.Test.ps1 => PipxPackages.Tests.ps1} (97%) diff --git a/images/win/scripts/Tests/PipxPackages.Test.ps1 b/images/win/scripts/Tests/PipxPackages.Tests.ps1 similarity index 97% rename from images/win/scripts/Tests/PipxPackages.Test.ps1 rename to images/win/scripts/Tests/PipxPackages.Tests.ps1 index b41b8f8f8..41041078a 100644 --- a/images/win/scripts/Tests/PipxPackages.Test.ps1 +++ b/images/win/scripts/Tests/PipxPackages.Tests.ps1 @@ -1,7 +1,7 @@ -Describe "PipxPackages" { - $pipxToolset = (Get-ToolsetContent).pipx - $testCases = $pipxToolset | ForEach-Object { @{package = $_.package; cmd = $_.cmd} } - It "" -TestCases $testCases { - "$cmd" | Should -ReturnZeroExitCode - } +Describe "PipxPackages" { + $pipxToolset = (Get-ToolsetContent).pipx + $testCases = $pipxToolset | ForEach-Object { @{package = $_.package; cmd = $_.cmd} } + It "" -TestCases $testCases { + "$cmd" | Should -ReturnZeroExitCode + } } \ No newline at end of file From 5694fe7952b2f2e733dd669d707e64384e966811 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Sun, 18 Oct 2020 23:10:10 +0300 Subject: [PATCH 07/23] fixed tests for windows 2016 --- images/win/toolsets/toolset-2016.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/win/toolsets/toolset-2016.json b/images/win/toolsets/toolset-2016.json index 011366bf3..197608e0d 100644 --- a/images/win/toolsets/toolset-2016.json +++ b/images/win/toolsets/toolset-2016.json @@ -242,7 +242,7 @@ "pipx": [ { "package": "yamllint", - "cmd": "yamllint" + "cmd": "yamllint --version" } ] } From b19a33bd3e27dce6430c7d7f266651a4c18dfb0a Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Wed, 21 Oct 2020 08:44:06 +0300 Subject: [PATCH 08/23] updated json files --- images/win/windows2016.json | 14 +++----------- images/win/windows2019.json | 14 +++----------- 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/images/win/windows2016.json b/images/win/windows2016.json index 0000e686c..120e7ac96 100644 --- a/images/win/windows2016.json +++ b/images/win/windows2016.json @@ -235,7 +235,9 @@ "{{ template_dir }}/scripts/Installers/Install-PyPy.ps1", "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1", "{{ template_dir }}/scripts/Installers/Configure-Toolset.ps1", - "{{ template_dir }}/scripts/Installers/Update-AndroidSDK.ps1" + "{{ template_dir }}/scripts/Installers/Update-AndroidSDK.ps1", + "{{ template_dir }}/scripts/Installers/Install-Pipx.ps1", + "{{ template_dir }}/scripts/Installers/Install-PipxPackages.ps1" ] }, { @@ -327,16 +329,6 @@ "{{ template_dir }}/scripts/Installers/Install-CodeQLBundle.ps1" ] }, - { - "type": "powershell", - "environment_vars": [ - "TOOLSET_JSON_PATH={{user `toolset_json_path`}}" - ], - "scripts": [ - "{{ template_dir }}/scripts/Installers/Install-Pipx.ps1", - "{{ template_dir }}/scripts/Installers/Install-PipxPackages.ps1" - ] - }, { "type": "powershell", "scripts": [ diff --git a/images/win/windows2019.json b/images/win/windows2019.json index fd32abb8c..997b1d43e 100644 --- a/images/win/windows2019.json +++ b/images/win/windows2019.json @@ -250,7 +250,9 @@ "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1", "{{ template_dir }}/scripts/Installers/Configure-Toolset.ps1", "{{ template_dir }}/scripts/Installers/Update-AndroidSDK.ps1", - "{{ template_dir }}/scripts/Installers/Install-AzureModules.ps1" + "{{ template_dir }}/scripts/Installers/Install-AzureModules.ps1", + "{{ template_dir }}/scripts/Installers/Install-Pipx.ps1", + "{{ template_dir }}/scripts/Installers/Install-PipxPackages.ps1" ] }, { @@ -325,16 +327,6 @@ "{{ template_dir }}/scripts/Installers/Install-CodeQLBundle.ps1" ] }, - { - "type": "powershell", - "environment_vars": [ - "TOOLSET_JSON_PATH={{user `toolset_json_path`}}" - ], - "scripts": [ - "{{ template_dir }}/scripts/Installers/Install-Pipx.ps1", - "{{ template_dir }}/scripts/Installers/Install-PipxPackages.ps1" - ] - }, { "type": "powershell", "scripts": [ From 77274dcb9b3f56947d632401eb612df84825d559 Mon Sep 17 00:00:00 2001 From: Nikita Bykov Date: Fri, 30 Oct 2020 12:36:09 +0300 Subject: [PATCH 09/23] updated Get-PipxVersion --- images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 b/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 index af9141e1d..6ee9fd5bc 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 @@ -283,5 +283,6 @@ function Get-YAMLLintVersion { } function Get-PipxVersion { - pipx --version + $pipxVersion = pipx --version + return "Pipx $pipxVersion" } \ No newline at end of file From 2faf574423319fe498d4ef69435c987585958db6 Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Mon, 2 Nov 2020 16:17:58 +0100 Subject: [PATCH 10/23] OSX: Rust tools: install 'rustfmt' and 'clippy' Make installed Rust components the same as for Ubuntu. See also: https://github.com/actions/virtual-environments/blob/f38833acec5cc1c3239474f8135b23c17cb72351/images/linux/scripts/installers/rust.sh --- images/macos/provision/core/rust.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/images/macos/provision/core/rust.sh b/images/macos/provision/core/rust.sh index de82c7b4a..89e70537c 100644 --- a/images/macos/provision/core/rust.sh +++ b/images/macos/provision/core/rust.sh @@ -11,7 +11,8 @@ CARGO_HOME=$HOME/.cargo source $CARGO_HOME/env echo Install common tools... +rustup component add rustfmt clippy cargo install bindgen cbindgen cargo-audit cargo-outdated echo Cleanup Cargo registry cached data... -rm -rf $CARGO_HOME/registry/* \ No newline at end of file +rm -rf $CARGO_HOME/registry/* From d511b0d2da53ec9a1fb9af3224d55607240da7b6 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Tue, 3 Nov 2020 17:58:18 +0500 Subject: [PATCH 11/23] [windows] give full access to Temp folder (#1959) Co-authored-by: Sergey Dolin --- images/win/scripts/Installers/Finalize-VM.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/images/win/scripts/Installers/Finalize-VM.ps1 b/images/win/scripts/Installers/Finalize-VM.ps1 index d0bb3d26a..8e29dbe20 100644 --- a/images/win/scripts/Installers/Finalize-VM.ps1 +++ b/images/win/scripts/Installers/Finalize-VM.ps1 @@ -33,4 +33,8 @@ Remove-Item $profile.AllUsersAllHosts -Force # Clean yarn and npm cache yarn cache clean -npm cache clean --force \ No newline at end of file +npm cache clean --force + +# allow msi to write to temp folder +# see https://github.com/actions/virtual-environments/issues/1704 +icacls "C:\Windows\Temp" /q /c /t /grant Users:F /T From aee4de77553ba46c02c81dbdab6c77faf6b1f4a9 Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Wed, 4 Nov 2020 09:23:31 +0300 Subject: [PATCH 12/23] deprecate old tools on MacOS --- images/macos/provision/core/node.sh | 4 ++-- images/macos/provision/core/xcode-tools.sh | 1 + images/macos/toolsets/toolset-10.15.json | 4 +--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/images/macos/provision/core/node.sh b/images/macos/provision/core/node.sh index 59e727ce9..194437a98 100644 --- a/images/macos/provision/core/node.sh +++ b/images/macos/provision/core/node.sh @@ -27,8 +27,8 @@ if is_Less_Catalina; then npm install -g appcenter-cli@^1.0.0 else # Install Node.JS 12 for macOS >= 10.15 - brew install node@12 - brew link node@12 --force + brew install node@14 + brew link node@14 --force for module in ${node_modules[@]}; do echo "Install $module" diff --git a/images/macos/provision/core/xcode-tools.sh b/images/macos/provision/core/xcode-tools.sh index 773b07a95..5c4917082 100755 --- a/images/macos/provision/core/xcode-tools.sh +++ b/images/macos/provision/core/xcode-tools.sh @@ -77,6 +77,7 @@ runFirstLaunch $DEFAULT_XCODE_VERSION if is_Catalina; then ln -sf /Applications/Xcode_11.2.1.app /Applications/Xcode_11.2.app ln -sf /Applications/Xcode_11.3.1.app /Applications/Xcode_11.3.app + ln -sf /Applications/Xcode_11.4.1.app /Applications/Xcode_11.4.app fi echo "Setting Xcode ${DEFAULT_XCODE_VERSION} as default" diff --git a/images/macos/toolsets/toolset-10.15.json b/images/macos/toolsets/toolset-10.15.json index 11a7e0161..79f6d9eb1 100644 --- a/images/macos/toolsets/toolset-10.15.json +++ b/images/macos/toolsets/toolset-10.15.json @@ -2,7 +2,7 @@ "xcode": { "default": "12", "versions": [ - "12.2_beta", "12.1", "12", "11.7", "11.6", "11.5", "11.4.1", "11.4", "11.3.1", "11.2.1", "11.1", "11", "10.3" + "12.2_beta", "12.1", "12", "11.7", "11.6", "11.5", "11.4.1", "11.3.1", "11.2.1", "10.3" ] }, "xamarin": { @@ -168,8 +168,6 @@ "arch": "x64", "platform" : "darwin", "versions": [ - "1.11.*", - "1.12.*", "1.13.*", "1.14.*", "1.15.*" From b8d7319f374456b52f375eac788368696a3fe732 Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Wed, 4 Nov 2020 09:27:56 +0300 Subject: [PATCH 13/23] deprecate old tools --- images/linux/toolsets/toolset-1604.json | 3 --- images/linux/toolsets/toolset-1804.json | 3 --- 2 files changed, 6 deletions(-) diff --git a/images/linux/toolsets/toolset-1604.json b/images/linux/toolsets/toolset-1604.json index 1136ae921..1c5ff5320 100644 --- a/images/linux/toolsets/toolset-1604.json +++ b/images/linux/toolsets/toolset-1604.json @@ -42,8 +42,6 @@ "arch": "x64", "platform" : "linux", "versions": [ - "1.11.*", - "1.12.*", "1.13.*", "1.14.*", "1.15.*" @@ -57,7 +55,6 @@ "platform_version": "16.04", "arch": "x64", "versions": [ - "1.69.0", "1.72.0" ] } diff --git a/images/linux/toolsets/toolset-1804.json b/images/linux/toolsets/toolset-1804.json index de611ebc0..226dd190a 100644 --- a/images/linux/toolsets/toolset-1804.json +++ b/images/linux/toolsets/toolset-1804.json @@ -42,8 +42,6 @@ "arch": "x64", "platform" : "linux", "versions": [ - "1.11.*", - "1.12.*", "1.13.*", "1.14.*", "1.15.*" @@ -57,7 +55,6 @@ "platform_version": "18.04", "arch": "x64", "versions": [ - "1.69.0", "1.72.0" ] } From 0ac8b88061567fedf8e5292b11bd09e6d57949cf Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Wed, 4 Nov 2020 09:30:19 +0300 Subject: [PATCH 14/23] deprecate old tools --- images/win/toolsets/toolset-2016.json | 1 - images/win/toolsets/toolset-2019.json | 10 ---------- 2 files changed, 11 deletions(-) diff --git a/images/win/toolsets/toolset-2016.json b/images/win/toolsets/toolset-2016.json index fef0a3fcb..8b6f32f36 100644 --- a/images/win/toolsets/toolset-2016.json +++ b/images/win/toolsets/toolset-2016.json @@ -73,7 +73,6 @@ "platform" : "win32", "toolset": "msvc14.1", "versions": [ - "1.69.0", "1.72.0" ] } diff --git a/images/win/toolsets/toolset-2019.json b/images/win/toolsets/toolset-2019.json index c76c9701b..a056fd88e 100644 --- a/images/win/toolsets/toolset-2019.json +++ b/images/win/toolsets/toolset-2019.json @@ -66,16 +66,6 @@ ], "default": "1.14.*" }, - { - "name": "Boost", - "url" : "https://raw.githubusercontent.com/actions/boost-versions/main/versions-manifest.json", - "arch": "x86_64", - "platform" : "win32", - "toolset": "msvc14.1", - "versions": [ - "1.69.0" - ] - }, { "name": "Boost", "url" : "https://raw.githubusercontent.com/actions/boost-versions/main/versions-manifest.json", From 2842ad4c0074b39cbb320a891acce1c50d086b45 Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov <47745270+al-cheb@users.noreply.github.com> Date: Wed, 4 Nov 2020 14:29:44 +0300 Subject: [PATCH 15/23] Revert "[Ubuntu] Install pwsh using dotnet tool" (#1979) --- images/linux/scripts/installers/powershellcore.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/images/linux/scripts/installers/powershellcore.sh b/images/linux/scripts/installers/powershellcore.sh index 4f700fbc9..0ac52072d 100644 --- a/images/linux/scripts/installers/powershellcore.sh +++ b/images/linux/scripts/installers/powershellcore.sh @@ -9,8 +9,7 @@ source $HELPER_SCRIPTS/os.sh # Install Powershell if isUbuntu20 ; then - dotnet tool install --tool-path /opt/microsoft/powershell/7 powershell - ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh + snap install powershell --classic --channel=edge/useedge fi if isUbuntu16 || isUbuntu18 ; then From 5fac6462098bc23c2c45728ced9120e74f064676 Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Thu, 5 Nov 2020 10:38:20 +0300 Subject: [PATCH 16/23] Update Node.Tests.ps1 --- images/macos/tests/Node.Tests.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images/macos/tests/Node.Tests.ps1 b/images/macos/tests/Node.Tests.ps1 index 7fde021d1..65005eb6e 100644 --- a/images/macos/tests/Node.Tests.ps1 +++ b/images/macos/tests/Node.Tests.ps1 @@ -4,7 +4,7 @@ Import-Module "$PSScriptRoot/../helpers/Tests.Helpers.psm1" Describe "Node.JS" { BeforeAll { $os = Get-OSVersion - $expectedNodeVersion = if ($os.IsHigherThanMojave) { "v12.*" } else { "v8.*" } + $expectedNodeVersion = if ($os.IsHigherThanMojave) { "v14.*" } else { "v8.*" } } It "Node.JS is installed" { @@ -29,7 +29,7 @@ Describe "NVM" { $nvmPath = Join-Path $env:HOME ".nvm" "nvm.sh" $nvmInitCommand = ". $nvmPath > /dev/null 2>&1 || true" } - + It "Nvm is installed" { $nvmPath | Should -Exist "$nvmInitCommand && nvm --version" | Should -ReturnZeroExitCode @@ -43,7 +43,7 @@ Describe "NVM" { param ( [string] $NvmVersion ) - + "$nvmInitCommand && nvm ls $($NvmVersion)" | Should -ReturnZeroExitCode } } From b326801306058663cf3b165f590e41214aad7e89 Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Thu, 5 Nov 2020 12:43:41 +0500 Subject: [PATCH 17/23] Print more informantion about cached Docker images on Windows (#1905) * Add docker full info * make tabular output * remove created at time Co-authored-by: Sergey Dolin --- .../SoftwareReport/SoftwareReport.Common.psm1 | 13 ++++++++++++- .../SoftwareReport/SoftwareReport.Generator.ps1 | 5 +++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 b/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 index 0ca95ee71..ea56e1ff2 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 @@ -262,6 +262,17 @@ function Get-CachedDockerImages { return (docker images --digests --format "* {{.Repository}}:{{.Tag}}").Split("*") | Where-Object { $_ } } +function Get-CachedDockerImagesTableData { + return (docker images --digests --format "*{{.Repository}}:{{.Tag}}|{{.Digest}} |{{.CreatedAt}}").Split("*") | Where-Object { $_ } | ForEach-Object { + $parts=$_.Split("|") + [PSCustomObject] @{ + "Repository:Tag" = $parts[0] + "Digest" = $parts[1] + "Created" = $parts[2].split(' ')[0] + } + } +} + function Get-PacmanVersion { $msys2BinDir = "C:\msys64\usr\bin" $pacmanPath = Join-Path $msys2BinDir "pacman.exe" @@ -280,4 +291,4 @@ function Get-ShellTarget { function Get-YAMLLintVersion { yamllint --version -} \ No newline at end of file +} diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 index 06e1054da..aba000b71 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 @@ -243,6 +243,7 @@ $markdown += New-MDNewLine # Docker images section $markdown += New-MDHeader "Cached Docker images" -Level 3 -$markdown += New-MDList -Style Unordered -Lines @(Get-CachedDockerImages) +$markdown += Get-CachedDockerImagesTableData | New-MDTable +$markdown += New-MDNewLine -$markdown | Out-File -FilePath "C:\InstalledSoftware.md" \ No newline at end of file +$markdown | Out-File -FilePath "C:\InstalledSoftware.md" From c45e72e08ee4197ccd283d231aef4b36fe5c80d5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2020 08:08:11 +0000 Subject: [PATCH 18/23] Updating readme file for ubuntu16 version 20201102.0 (#1968) Co-authored-by: Image generation service account Co-authored-by: Actions service account --- images/linux/Ubuntu1604-README.md | 52 ++++++++++++++++--------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/images/linux/Ubuntu1604-README.md b/images/linux/Ubuntu1604-README.md index c06b5d7b9..57198a444 100644 --- a/images/linux/Ubuntu1604-README.md +++ b/images/linux/Ubuntu1604-README.md @@ -1,11 +1,13 @@ | Announcements | |-| +| [Default Node.JS will be switched to 14.x on all platforms ](https://github.com/actions/virtual-environments/issues/1953) | +| [[macOS] Default Python will be upgraded to 3.9](https://github.com/actions/virtual-environments/issues/1929) | | [.NET 5.0 will become a default .NET version on November, 10](https://github.com/actions/virtual-environments/issues/1891) | | [[Ubuntu] [Windows] Boost 1.69.0 will be deprecated on November, 10](https://github.com/actions/virtual-environments/issues/1847) | | [Ubuntu-latest workflows will use Ubuntu-20.04](https://github.com/actions/virtual-environments/issues/1816) | *** # Ubuntu 16.04.7 LTS -- Image Version: 20201026.1 +- Image Version: 20201102.0 ## Installed Software ### Language and Runtime @@ -14,7 +16,7 @@ - Clang 6.0.0, 8.0.0, 9.0.1 - Erlang 11.1 - Mono 6.12.0.90 -- Node 12.19.0 +- Node 14.15.0 - Python 2.7.12 - Python3 3.5.2 - PowerShell 7.0.3 @@ -24,8 +26,8 @@ ### Package Management - Gem 3.1.4 -- Helm 3.3.4 -- Homebrew 2.5.6 +- Helm 3.4.0 +- Homebrew 2.5.8 - Miniconda 4.8.3 - Npm 6.14.8 - Pip 8.1.1 @@ -48,7 +50,7 @@ - Bazel 3.7.0 - Bazelisk 1.7.3 - CMake 3.17.0 -- CodeQL Action Bundle 2.3.0 +- CodeQL Action Bundle 2.3.1+202010222007 - curl 7.47.0 - Docker Compose 1.27.4 - Docker-Buildx 0.4.2 @@ -56,21 +58,21 @@ - Git 2.29.0 - Git LFS 2.12.0 - Git-ftp 1.0.2 -- Google Cloud SDK 315.0.0 +- Google Cloud SDK 316.0.0 - Haveged 1.9.1 -- Heroku 7.46.2 +- Heroku 7.47.0 - HHVM (HipHop VM) 4.56.1 - jq 1.5 - Kind 0.9.0 - Kubectl 1.19.3 -- Kustomize 3.8.5 +- Kustomize 3.8.6 - Leiningen 2.9.4 - m4 1.4.17 - Mercurial 4.4.1 -- Minikube 1.14.1 -- Newman 5.2.0 +- Minikube 1.14.2 +- Newman 5.2.1 - nvm 0.36.0 -- Packer 1.6.4 +- Packer 1.6.5 - PhantomJS 2.1.1 - Pulumi 2.12.1 - R 4.0.3 @@ -86,14 +88,14 @@ ### CLI Tools - Alibaba Cloud CLI 3.0.60 -- AWS CLI 1.18.164 +- AWS CLI 1.18.169 - AWS CLI Session manager plugin 1.2.7.0 -- AWS SAM CLI 1.6.2 -- Azure CLI (azure-cli) 2.13.0 +- AWS SAM CLI 1.7.0 +- Azure CLI (azure-cli) 2.14.0 - Azure CLI (azure-devops) 0.18.0 -- GitHub CLI 1.1.0 +- GitHub CLI 1.2.0 - Hub CLI 2.14.2 -- Netlify CLI 2.65.7 +- Netlify CLI 2.67.2 - oc CLI 4.5.0 - ORAS CLI 0.8.1 - Vercel CLI 20.1.2 @@ -109,8 +111,8 @@ ### PHP | Tool | Version | | -------- | ----------------------------------------- | -| PHP | 5.6.40 7.0.33 7.1.33 7.2.34 7.3.23 7.4.11 | -| Composer | 2.0.2 | +| PHP | 5.6.40 7.0.33 7.1.33 7.2.34 7.3.24 7.4.12 | +| Composer | 2.0.4 | | PHPUnit | 7.5.20 | ### Haskell @@ -126,16 +128,16 @@ #### Packages - Bindgen 0.55.1 -- Cargo audit 0.12.1 -- Cargo outdated 0.9.11 +- Cargo audit 0.13.1 +- Cargo outdated 0.9.13 - Cargo clippy 0.0.212 - Cbindgen 0.15.0 - Rustfmt 1.4.20 ### Browsers and Drivers -- Google Chrome 86.0.4240.111 +- Google Chrome 86.0.4240.183 - ChromeDriver 86.0.4240.22 -- Mozilla Firefox 81.0.2 +- Mozilla Firefox 82.0 - Geckodriver 0.27.0 ### .NET Core SDK @@ -150,7 +152,7 @@ - sqlite3 3.11.0 #### MySQL -- MySQL 5.7.31 +- MySQL 5.7.32 - MySQL Server (user:root password:root) - MS SQL Server Client Tools @@ -178,9 +180,9 @@ #### Node.js - 8.17.0 -- 10.22.1 +- 10.23.0 - 12.19.0 -- 14.14.0 +- 14.15.0 #### Go - 1.11.13 From 5c99d59f22fdacc0fce92863f037421b936e7f42 Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov <47745270+al-cheb@users.noreply.github.com> Date: Thu, 5 Nov 2020 11:37:02 +0300 Subject: [PATCH 19/23] Revert "[windows] add bash wrapper from git (#1937)" This reverts commit 8798c75d674144ebee7cd2c15224318697b442f6. --- images/win/scripts/Installers/Configure-Shell.ps1 | 4 ++-- images/win/scripts/Installers/Install-Msys2.ps1 | 6 ------ images/win/scripts/Tests/Shell.Tests.ps1 | 4 ++-- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/images/win/scripts/Installers/Configure-Shell.ps1 b/images/win/scripts/Installers/Configure-Shell.ps1 index 61b30bf3b..8eb163bcf 100644 --- a/images/win/scripts/Installers/Configure-Shell.ps1 +++ b/images/win/scripts/Installers/Configure-Shell.ps1 @@ -3,8 +3,8 @@ $shellPath = "C:\shells" New-Item -Path $shellPath -ItemType Directory | Out-Null # sh and bash <--> C:\msys64\usr\bin\bash.exe -New-Item -ItemType SymbolicLink -Path "$shellPath\bash.exe" -Target "C:\msys64\bin\bash.exe" | Out-Null -New-Item -ItemType SymbolicLink -Path "$shellPath\sh.exe" -Target "C:\msys64\bin\sh.exe" | Out-Null +New-Item -ItemType SymbolicLink -Path "$shellPath\bash.exe" -Target "C:\msys64\usr\bin\bash.exe" | Out-Null +New-Item -ItemType SymbolicLink -Path "$shellPath\sh.exe" -Target "C:\msys64\usr\bin\sh.exe" | Out-Null # WSL is available on Windows Server 2019 if (Test-IsWin19) diff --git a/images/win/scripts/Installers/Install-Msys2.ps1 b/images/win/scripts/Installers/Install-Msys2.ps1 index 26a61e39d..dea3b95d0 100644 --- a/images/win/scripts/Installers/Install-Msys2.ps1 +++ b/images/win/scripts/Installers/Install-Msys2.ps1 @@ -95,10 +95,4 @@ if (Test-Path "C:\Program Files\Git\etc\ssh") ssh-keyscan -t rsa ssh.dev.azure.com >> "C:\Program Files\Git\etc\ssh\ssh_known_hosts" } -# Copy bash wrapper from git -$wrapperPath = "C:\msys64\bin" -New-Item -Path $wrapperPath -ItemType Directory -Force | Out-Null -Copy-Item -Path "$env:ProgramFiles\Git\bin\bash.exe" -Destination $wrapperPath -Copy-Item -Path "$env:ProgramFiles\Git\bin\sh.exe" -Destination $wrapperPath - Invoke-PesterTests -TestFile "MSYS2" diff --git a/images/win/scripts/Tests/Shell.Tests.ps1 b/images/win/scripts/Tests/Shell.Tests.ps1 index 084f40cad..915940e2a 100644 --- a/images/win/scripts/Tests/Shell.Tests.ps1 +++ b/images/win/scripts/Tests/Shell.Tests.ps1 @@ -1,7 +1,7 @@ Describe "Shell" { $shellTestCases = @( - @{Name = "C:\shells\bash.exe"; Target = "C:\msys64\bin\bash.exe"}, - @{Name = "C:\shells\sh.exe"; Target = "C:\msys64\bin\sh.exe"}, + @{Name = "C:\shells\bash.exe"; Target = "C:\msys64\usr\bin\bash.exe"}, + @{Name = "C:\shells\sh.exe"; Target = "C:\msys64\usr\bin\sh.exe"}, @{Name = "C:\shells\gitbash.exe"; Target = "$env:ProgramFiles\Git\bin\bash.exe"}, @{Name = "C:\shells\msysbash.exe"; Target = "C:\msys64\usr\bin\bash.exe"} ) From 635a57ab889946079dc4bfaa3850f27ff6308411 Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov <47745270+al-cheb@users.noreply.github.com> Date: Thu, 5 Nov 2020 11:39:54 +0300 Subject: [PATCH 20/23] Revert "[Windows] Removed Git internal tools from PATH" --- .../scripts/Installers/Configure-Shell.ps1 | 23 -------- images/win/scripts/Installers/Install-Git.ps1 | 12 +++- .../scripts/Installers/Install-Mingw64.ps1 | 2 +- .../win/scripts/Installers/Install-Msys2.ps1 | 55 ++++++++----------- .../SoftwareReport/SoftwareReport.Common.psm1 | 7 --- .../SoftwareReport.Generator.ps1 | 10 +--- images/win/scripts/Tests/Git.Tests.ps1 | 2 +- images/win/scripts/Tests/Shell.Tests.ps1 | 50 ----------------- images/win/windows2016.json | 3 +- images/win/windows2019.json | 3 +- 10 files changed, 40 insertions(+), 127 deletions(-) delete mode 100644 images/win/scripts/Installers/Configure-Shell.ps1 delete mode 100644 images/win/scripts/Tests/Shell.Tests.ps1 diff --git a/images/win/scripts/Installers/Configure-Shell.ps1 b/images/win/scripts/Installers/Configure-Shell.ps1 deleted file mode 100644 index 8eb163bcf..000000000 --- a/images/win/scripts/Installers/Configure-Shell.ps1 +++ /dev/null @@ -1,23 +0,0 @@ -# Create shells folder -$shellPath = "C:\shells" -New-Item -Path $shellPath -ItemType Directory | Out-Null - -# sh and bash <--> C:\msys64\usr\bin\bash.exe -New-Item -ItemType SymbolicLink -Path "$shellPath\bash.exe" -Target "C:\msys64\usr\bin\bash.exe" | Out-Null -New-Item -ItemType SymbolicLink -Path "$shellPath\sh.exe" -Target "C:\msys64\usr\bin\sh.exe" | Out-Null - -# WSL is available on Windows Server 2019 -if (Test-IsWin19) -{ - # winbash <--> C:\Windows\System32\bash.exe - New-Item -ItemType SymbolicLink -Path "$shellPath\winbash.exe" -Target "$env:SystemRoot\System32\bash.exe" | Out-Null -} - -# gitbash <--> C:\Program Files\Git\bin\bash.exe -New-Item -ItemType SymbolicLink -Path "$shellPath\gitbash.exe" -Target "$env:ProgramFiles\Git\bin\bash.exe" | Out-Null - -# msysbash <--> C:\msys64\usr\bin\bash.exe -New-Item -ItemType SymbolicLink -Path "$shellPath\msysbash.exe" -Target "C:\msys64\usr\bin\bash.exe" | Out-Null - -# Add shells to PATH -Add-MachinePathItem $shellPath \ No newline at end of file diff --git a/images/win/scripts/Installers/Install-Git.ps1 b/images/win/scripts/Installers/Install-Git.ps1 index 6c89250e9..27a619695 100644 --- a/images/win/scripts/Installers/Install-Git.ps1 +++ b/images/win/scripts/Installers/Install-Git.ps1 @@ -26,7 +26,7 @@ Install-Binary -Url $downloadUrl ` "/SP-", ` "/CLOSEAPPLICATIONS", ` "/RESTARTAPPLICATIONS", ` - "/o:PathOption=Cmd", ` + "/o:PathOption=CmdTools", ` "/o:BashTerminalOption=ConHost", ` "/o:EnableSymlinks=Enabled", ` "/COMPONENTS=gitlfs") @@ -36,5 +36,15 @@ Choco-Install -PackageName hub # Disable GCM machine-wide [Environment]::SetEnvironmentVariable("GCM_INTERACTIVE", "Never", [System.EnvironmentVariableTarget]::Machine) +Add-MachinePathItem "C:\Program Files\Git\bin" + +if (Test-IsWin16) { + $env:Path += ";$env:ProgramFiles\Git\usr\bin\" +} + +# Add well-known SSH host keys to ssh_known_hosts +ssh-keyscan -t rsa github.com >> "C:\Program Files\Git\etc\ssh\ssh_known_hosts" +ssh-keyscan -t rsa ssh.dev.azure.com >> "C:\Program Files\Git\etc\ssh\ssh_known_hosts" + Invoke-PesterTests -TestFile "Git" -TestName "Git" Invoke-PesterTests -TestFile "CLI.Tools" -TestName "Hub CLI" diff --git a/images/win/scripts/Installers/Install-Mingw64.ps1 b/images/win/scripts/Installers/Install-Mingw64.ps1 index 2b999f450..b6c860485 100644 --- a/images/win/scripts/Installers/Install-Mingw64.ps1 +++ b/images/win/scripts/Installers/Install-Mingw64.ps1 @@ -7,7 +7,7 @@ Choco-Install -PackageName mingw # Make a copy of mingw32-make.exe to make.exe, which is a more discoverable name # and so the same command line can be used on Windows as on macOS and Linux -$path = Get-Command mingw32-make.exe -CommandType All | Where-Object { $_.Path.Contains("C:\ProgramData\Chocolatey") } | Get-Item +$path = where.exe mingw32-make.exe | Get-Item Copy-Item -Path $path -Destination (Join-Path $path.Directory 'make.exe') Invoke-PesterTests -TestFile "Tools" -TestName "Mingw64" diff --git a/images/win/scripts/Installers/Install-Msys2.ps1 b/images/win/scripts/Installers/Install-Msys2.ps1 index dea3b95d0..f0da38503 100644 --- a/images/win/scripts/Installers/Install-Msys2.ps1 +++ b/images/win/scripts/Installers/Install-Msys2.ps1 @@ -9,19 +9,35 @@ $dash = "-" * 40 -# Downloading msys2 -$msys2Release = "https://api.github.com/repos/msys2/msys2-installer/releases/latest" -$msys2Uri = ((Invoke-RestMethod $msys2Release).assets | Where-Object { - $_.name -match "x86_64" -and $_.name.EndsWith("sfx.exe") }).browser_download_url -$msys2File = Start-DownloadWithRetry -Url $msys2Uri +$origPath = $env:PATH +$gitPath = "$env:ProgramFiles\Git" -# extract sfx.exe to C:\ +$msys2_release = "https://api.github.com/repos/msys2/msys2-installer/releases/latest" + +$msys2Uri = ((Invoke-RestMethod $msys2_release).assets | Where-Object { + $_.name -match "x86_64" -and $_.name.EndsWith("tar.xz") }).browser_download_url + +# Download the latest msys2 x86_64, filename includes release date +Write-Host "Starting msys2 download using $($msys2Uri.split('/')[-1])" +$msys2File = Start-DownloadWithRetry -Url $msys2Uri +Write-Host "Finished download" + +# nix style path for tar +$msys2FileU = "/$msys2File".replace(':', '').replace('\', '/') + +# Git tar needs exe's from mingw64\bin +$env:PATH = "$gitPath\usr\bin;$gitPath\mingw64\bin;$origPath" + +$tar = "$gitPath\usr\bin\tar.exe" + +# extract tar.xz to C:\ Write-Host "Starting msys2 extraction" -& $msys2File -y -oC:\ +&$tar -xJf $msys2FileU -C /c/ +Remove-Item $msys2File Write-Host "Finished extraction" # Add msys2 bin tools folders to PATH temporary -$env:PATH = "C:\msys64\mingw64\bin;C:\msys64\usr\bin;$env:PATH" +$env:PATH = "C:\msys64\mingw64\bin;C:\msys64\usr\bin;$origPath" Write-Host "`n$dash bash pacman-key --init" bash.exe -c "pacman-key --init 2>&1" @@ -57,10 +73,6 @@ Write-Host "`n$dash Install mingw32 packages" $pre = "mingw-w64-i686-" pacman.exe -S --noconfirm --needed --noprogressbar $tools32.replace('___', $pre).split(' ') -# install openssh -Write-Host "`n$dash Install openssh package" -pacman.exe -S --noconfirm --needed --noprogressbar openssh - # clean all packages to decrease image size Write-Host "`n$dash Clean packages" pacman.exe -Scc --noconfirm @@ -76,23 +88,4 @@ pacman.exe -Q | grep -v ^mingw-w64- Write-Host "`nMSYS2 installation completed" -# Environment -# add C:\msys64\mingw64\bin and C:\msys64\usr\bin to PATH -# C:\msys64\mingw64\bin add after C:\Windows\System32 to not replace built-in tar.exe -$regEnvKey = 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -$pathValue = Get-ItemPropertyValue -Path $regEnvKey -Name 'Path' -$pathValue += ";C:\msys64\mingw64\bin;C:\msys64\usr\bin" -Set-ItemProperty -Path $regEnvKey -Name 'Path' -Value $pathValue - -# Add well-known SSH host keys to ssh_known_hosts to Msys2 -ssh-keyscan -t rsa github.com >> "C:\msys64\etc\ssh\ssh_known_hosts" -ssh-keyscan -t rsa ssh.dev.azure.com >> "C:\msys64\etc\ssh\ssh_known_hosts" - -# Add well-known SSH host keys to ssh_known_hosts to Git -if (Test-Path "C:\Program Files\Git\etc\ssh") -{ - ssh-keyscan -t rsa github.com >> "C:\Program Files\Git\etc\ssh\ssh_known_hosts" - ssh-keyscan -t rsa ssh.dev.azure.com >> "C:\Program Files\Git\etc\ssh\ssh_known_hosts" -} - Invoke-PesterTests -TestFile "MSYS2" diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 b/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 index 72a336193..be9603f62 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Common.psm1 @@ -282,13 +282,6 @@ function Get-PacmanVersion { return "- Pacman $pacmanVersion" } -function Get-ShellTarget { - $shells = Get-ChildItem C:\shells -File | Select-Object @{n="Name";e={ - $name = $_.Name - if ($name -eq 'bash.exe') {"$name (Default)"} else {$name}}},@{n="Target";e={@($_.Target)[0]}} | Sort-Object Name - $shells | New-MDTable -Columns ([ordered]@{Name = "left"; Target = "left";}) -} - function Get-YAMLLintVersion { yamllint --version } diff --git a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 index fff45919a..91c19dc6f 100644 --- a/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 +++ b/images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1 @@ -143,22 +143,14 @@ $markdown += New-MDList -Style Unordered -Lines @( (Get-SeleniumWebDriverVersion -Driver "iexplorer") ) -$markdown += New-MDHeader "Shells" -Level 3 -$markdown += Get-ShellTarget -$markdown += New-MDNewLine - $markdown += New-MDHeader "MSYS2" -Level 3 $markdown += Get-PacmanVersion $markdown += New-MDNewLine -$markdown += New-MDHeader "Notes:" -Level 5 $markdown += @' ``` Location: C:\msys64 -1. MSYS2 is pre-installed on image -2. C:\msys64\mingw64\bin is added to PATH and has lower precedence than C:\Windows\System32 -3. C:\msys64\usr\bin is added to PATH and has lower precedence than C:\Windows\System32 -4. Default bash.exe shell is set to the C:\msys64\usr\bin\bash.exe +Note: MSYS2 is pre-installed on image but not added to PATH. ``` '@ $markdown += New-MDNewLine diff --git a/images/win/scripts/Tests/Git.Tests.ps1 b/images/win/scripts/Tests/Git.Tests.ps1 index e7675ee07..8dd159fa4 100644 --- a/images/win/scripts/Tests/Git.Tests.ps1 +++ b/images/win/scripts/Tests/Git.Tests.ps1 @@ -1,5 +1,5 @@ Describe "Git" { - $gitTools = 'git', 'git-lfs' + $gitTools = 'bash', 'awk', 'git', 'git-lfs' $gitTestCases = $gitTools | ForEach-Object { @{ toolName = $_ diff --git a/images/win/scripts/Tests/Shell.Tests.ps1 b/images/win/scripts/Tests/Shell.Tests.ps1 deleted file mode 100644 index 915940e2a..000000000 --- a/images/win/scripts/Tests/Shell.Tests.ps1 +++ /dev/null @@ -1,50 +0,0 @@ -Describe "Shell" { - $shellTestCases = @( - @{Name = "C:\shells\bash.exe"; Target = "C:\msys64\usr\bin\bash.exe"}, - @{Name = "C:\shells\sh.exe"; Target = "C:\msys64\usr\bin\sh.exe"}, - @{Name = "C:\shells\gitbash.exe"; Target = "$env:ProgramFiles\Git\bin\bash.exe"}, - @{Name = "C:\shells\msysbash.exe"; Target = "C:\msys64\usr\bin\bash.exe"} - ) - - $pathTestCases = @( - @{Path = "C:\shells"}, - @{Path = "C:\msys64\mingw64\bin"}, - @{Path = "C:\msys64\usr\bin"} - ) - - $IsWin16 = Test-IsWin16 - - It "Default bash.exe from MSYS2" { - (Get-Command bash).Path | Should -BeExactly "C:\shells\bash.exe" - } - - It "Default sh.exe from MSYS2" { - (Get-Command sh).Path | Should -BeExactly "C:\shells\sh.exe" - } - - It "Folder C:\shells exists" { - "C:\shells" | Should -Exist - } - - It "C:\Windows\System32 before C:\msys64\mingw64\bin and C:\msys64\usr\bin" { - $path = $env:Path.Split(";").ToLower() - $indexOfSystem32 = $path.IndexOf("c:\windows\system32") - - $path.IndexOf("c:\msys64\mingw64\bin") | Should -BeGreaterThan $indexOfSystem32 - $path.IndexOf("c:\msys64\usr\bin") | Should -BeGreaterThan $indexOfSystem32 - } - - It "C:\shells\winbash.exe target to $env:SystemRoot\System32\bash.exe" -Skip:$IsWin16 { - $Name = "C:\shells\winbash.exe" - $Target = "$env:SystemRoot\System32\bash.exe" - (Get-Item $Name).Target | Should -BeExactly $Target - } - - It " target to " -TestCases $shellTestCases { - (Get-Item $Name).Target | Should -BeExactly $Target - } - - It " is in PATH" -TestCases $pathTestCases { - $env:Path.Split(";") | Should -Contain $Path - } -} \ No newline at end of file diff --git a/images/win/windows2016.json b/images/win/windows2016.json index 8136ebd4d..120e7ac96 100644 --- a/images/win/windows2016.json +++ b/images/win/windows2016.json @@ -333,8 +333,7 @@ "type": "powershell", "scripts": [ "{{ template_dir }}/scripts/Installers/Install-WindowsUpdates.ps1", - "{{ template_dir }}/scripts/Installers/Configure-DynamicPort.ps1", - "{{ template_dir }}/scripts/Installers/Configure-Shell.ps1" + "{{ template_dir }}/scripts/Installers/Configure-DynamicPort.ps1" ], "elevated_user": "{{user `install_user`}}", "elevated_password": "{{user `install_password`}}" diff --git a/images/win/windows2019.json b/images/win/windows2019.json index f20e050c6..a27bf27d9 100644 --- a/images/win/windows2019.json +++ b/images/win/windows2019.json @@ -331,8 +331,7 @@ "type": "powershell", "scripts": [ "{{ template_dir }}/scripts/Installers/Install-WindowsUpdates.ps1", - "{{ template_dir }}/scripts/Installers/Configure-DynamicPort.ps1", - "{{ template_dir }}/scripts/Installers/Configure-Shell.ps1" + "{{ template_dir }}/scripts/Installers/Configure-DynamicPort.ps1" ], "elevated_user": "{{user `install_user`}}", "elevated_password": "{{user `install_password`}}" From ff490685427ede9fc41776d07d6b9ac33b34aa9c Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Thu, 5 Nov 2020 14:20:40 +0500 Subject: [PATCH 21/23] Print more informantion about cached Docker images linux (#1906) * Add docker full info * make tabular output * fix shebang * remove created at time Co-authored-by: Sergey Dolin --- .../SoftwareReport/SoftwareReport.Common.psm1 | 13 ++++++++++++- .../SoftwareReport/SoftwareReport.Generator.ps1 | 3 ++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/images/linux/scripts/SoftwareReport/SoftwareReport.Common.psm1 b/images/linux/scripts/SoftwareReport/SoftwareReport.Common.psm1 index 1989caf4b..15a19653c 100644 --- a/images/linux/scripts/SoftwareReport/SoftwareReport.Common.psm1 +++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Common.psm1 @@ -240,6 +240,17 @@ function Get-CachedDockerImages { return $images } +function Get-CachedDockerImagesTableData { + return (sudo docker images --digests --format "*{{.Repository}}:{{.Tag}}|{{.Digest}} |{{.CreatedAt}}").Split("*") | Where-Object { $_ } | ForEach-Object { + $parts=$_.Split("|") + [PSCustomObject] @{ + "Repository:Tag" = $parts[0] + "Digest" = $parts[1] + "Created" = $parts[2].split(' ')[0] + } + } +} + function Get-AptPackages { $toolsetJson = Get-ToolsetContent $apt = $toolsetJson.apt @@ -252,4 +263,4 @@ function Get-PipxVersion { $result -match "(?\d+\.\d+\.\d+\.?\d*)" | Out-Null $pipxVersion = $Matches.Version return "Pipx $pipxVersion" -} \ No newline at end of file +} diff --git a/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 b/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 index df77ea23b..9cf57034a 100644 --- a/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 +++ b/images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1 @@ -224,7 +224,8 @@ $markdown += Build-AndroidTable | New-MDTable $markdown += New-MDNewLine $markdown += New-MDHeader "Cached Docker images" -Level 3 -$markdown += New-MDList -Style Unordered -Lines @(Get-CachedDockerImages) +$markdown += Get-CachedDockerImagesTableData | New-MDTable +$markdown += New-MDNewLine $markdown += New-MDHeader "Installed apt packages" -Level 3 $markdown += New-MDList -Style Unordered -Lines @(Get-AptPackages) From ad583eba90bab0332a4cc322bbfedad21a901312 Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov Date: Thu, 5 Nov 2020 15:53:49 +0300 Subject: [PATCH 22/23] add reboot for Ubuntu20 --- images/linux/scripts/base/apt.sh | 6 +----- images/linux/scripts/base/repos.sh | 5 ++++- images/linux/ubuntu1604.json | 3 +++ images/linux/ubuntu1804.json | 3 +++ images/linux/ubuntu2004.json | 11 +++++++++++ 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/images/linux/scripts/base/apt.sh b/images/linux/scripts/base/apt.sh index 1efbb6721..cf221452c 100644 --- a/images/linux/scripts/base/apt.sh +++ b/images/linux/scripts/base/apt.sh @@ -1,9 +1,5 @@ #!/bin/bash -e -export DEBIAN_FRONTEND=noninteractive -apt-get -yq update -apt-get -yq dist-upgrade - # Stop and disable apt-daily upgrade services; systemctl stop apt-daily.timer systemctl disable apt-daily.timer @@ -16,7 +12,7 @@ systemctl disable apt-daily-upgrade.service sudo sed -i 's/APT::Periodic::Update-Package-Lists "1"/APT::Periodic::Update-Package-Lists "0"/' /etc/apt/apt.conf.d/20auto-upgrades # Enable retry logic for apt up to 10 times -echo "APT::Acquire::Retries \"10\";" > /etc/apt/apt.conf.d/80-retries +echo "APT::Acquire::Retries \"10\";" > /etc/apt/apt.conf.d/80-retries # Configure apt to always assume Y echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/90assumeyes diff --git a/images/linux/scripts/base/repos.sh b/images/linux/scripts/base/repos.sh index 9005d32aa..a489c62ae 100644 --- a/images/linux/scripts/base/repos.sh +++ b/images/linux/scripts/base/repos.sh @@ -16,4 +16,7 @@ curl -L https://packages.microsoft.com/keys/microsoft.asc | apt-key add - curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg -apt-get update \ No newline at end of file + +# update +apt-get -yq update +apt-get -yq dist-upgrade diff --git a/images/linux/ubuntu1604.json b/images/linux/ubuntu1604.json index a519b30d9..6168f4bf8 100644 --- a/images/linux/ubuntu1604.json +++ b/images/linux/ubuntu1604.json @@ -72,6 +72,9 @@ { "type": "shell", "script": "{{template_dir}}/scripts/base/apt.sh", + "environment_vars": [ + "DEBIAN_FRONTEND=noninteractive" + ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, { diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index 977464e3a..6fc3f813f 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -75,6 +75,9 @@ { "type": "shell", "script": "{{template_dir}}/scripts/base/apt.sh", + "environment_vars": [ + "DEBIAN_FRONTEND=noninteractive" + ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, { diff --git a/images/linux/ubuntu2004.json b/images/linux/ubuntu2004.json index 54bc6fb57..bde4024c0 100644 --- a/images/linux/ubuntu2004.json +++ b/images/linux/ubuntu2004.json @@ -74,9 +74,20 @@ ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, + { + "type": "shell", + "expect_disconnect": true, + "scripts": [ + "{{template_dir}}/scripts/base/reboot.sh" + ], + "execute_command": "/bin/sh -c '{{ .Vars }} {{ .Path }}'" + }, { "type": "shell", "script": "{{template_dir}}/scripts/base/apt.sh", + "environment_vars": [ + "DEBIAN_FRONTEND=noninteractive" + ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, { From 567e92c4dc955fe8d19390b241f5e3467bb7d7e8 Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov <47745270+al-cheb@users.noreply.github.com> Date: Thu, 5 Nov 2020 23:01:06 +0300 Subject: [PATCH 23/23] [Ubuntu] Freeze Az.Accounts module for Ubuntu 20 (#1993) * add reboot for Ubuntu20 * freeze Az.Accounts 1.9.5 --- images/linux/scripts/installers/azpowershell.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/images/linux/scripts/installers/azpowershell.sh b/images/linux/scripts/installers/azpowershell.sh index 13253b5fa..f18492c5e 100644 --- a/images/linux/scripts/installers/azpowershell.sh +++ b/images/linux/scripts/installers/azpowershell.sh @@ -18,6 +18,10 @@ pwsh -Command "Update-Module -Name PowerShellGet -Force" # Install Azure CLI (instructions taken from https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) for version in ${versions[@]}; do pwsh -Command "Save-Module -Name Az -LiteralPath /usr/share/az_$version -RequiredVersion $version -Force -Verbose" + if isUbuntu20; then + rm -rf "/usr/share/az_$version/Az.Accounts" + pwsh -Command "Save-Module -Name Az.Accounts -LiteralPath /usr/share/az_$version -RequiredVersion 1.9.5 -Force -Verbose" + fi done # Run tests to determine that the software installed as expected