From 5a74642eaba226aed28de70aa2a9da5f2d61a2a5 Mon Sep 17 00:00:00 2001 From: Jason Edstrom Date: Tue, 4 Feb 2020 13:34:46 -0600 Subject: [PATCH 01/67] Updated maven from 3.6.2 to 3.6.3 --- images/win/scripts/Installers/Install-JavaTools.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/win/scripts/Installers/Install-JavaTools.ps1 b/images/win/scripts/Installers/Install-JavaTools.ps1 index 7b8cdd135..68c0209fd 100644 --- a/images/win/scripts/Installers/Install-JavaTools.ps1 +++ b/images/win/scripts/Installers/Install-JavaTools.ps1 @@ -62,7 +62,7 @@ setx JAVA_HOME_11_X64 $latestJava11Install /M # Install Java tools # Force chocolatey to ignore dependencies on Ant and Maven or else they will download the Oracle JDK choco install ant -y -i -choco install maven -y -i --version=3.6.2 +choco install maven -y -i --version=3.6.3 choco install gradle -y # Move maven variables to Machine. They may not be in the environment for this script so we need to read them from the registry. From b98eec89d4711feb96541bd67fa45cd79e55d180 Mon Sep 17 00:00:00 2001 From: Sascha Mann Date: Sun, 9 Feb 2020 19:37:43 +0100 Subject: [PATCH 02/67] Add Julia 1.3.1 --- images/linux/scripts/installers/julia.sh | 50 +++++++++++++++ .../win/scripts/Installers/Install-Julia.ps1 | 62 +++++++++++++++++++ .../win/scripts/Installers/Validate-Julia.ps1 | 49 +++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 images/linux/scripts/installers/julia.sh create mode 100644 images/win/scripts/Installers/Install-Julia.ps1 create mode 100644 images/win/scripts/Installers/Validate-Julia.ps1 diff --git a/images/linux/scripts/installers/julia.sh b/images/linux/scripts/installers/julia.sh new file mode 100644 index 000000000..849ff8f29 --- /dev/null +++ b/images/linux/scripts/installers/julia.sh @@ -0,0 +1,50 @@ +#!/bin/bash +################################################################################ +## File: julia.sh +## Desc: Installs Julia, and adds Julia to the path +################################################################################ + +# Source the helpers for use with the script +source $HELPER_SCRIPTS/document.sh + +# This function installs Julia using the specified arguments: +# $1=MajorVersion (1.3) +# $2=MajorAndMinorVersion (1.3.1) +# $3=IsDefaultVersion (true or false) + +function InstallJulia () { + curl -sL "https://julialang-s3.julialang.org/bin/linux/x64/$1/julia-$2-linux-x86_64.tar.gz" -o "julia-$2-linux-x86_64.tar.gz" + mkdir -p "/usr/local/julia$1" + tar -C "/usr/local/julia$1" -xzf "julia-$2-linux-x86_64.tar.gz" --strip-components=1 julia + rm "julia-$2-linux-x86_64.tar.gz" + + # If this version of Julia is to be the default version, + # symlink it into the path + if [ "$3" = true ]; then + ln -s "/usr/local/julia$1/bin/julia" /usr/bin/julia + fi + + # Run tests to determine that the software installed as expected + echo "Testing to make sure that script performed as expected, and basic scenarios work" + + # If this version of Julia is to be the default version, + # check that it has been added to PATH + if [ "$3" = true ]; then + if ! command -v julia; then + echo "Julia was not installed or found on PATH" + exit 1 + fi + fi + + # Verify output of julia --version + if [ ! "$(/usr/local/julia"$1"/bin/julia --version)" = "julia version $2" ]; then + echo "Julia was not installed correctly" + exit 1 + fi + + # Document what was added to the image + echo "Lastly, documenting what we added to the metadata file" + DocumentInstalledItem "Julia ($(julia --version))" +} + +InstallJulia 1.3 1.3.1 true diff --git a/images/win/scripts/Installers/Install-Julia.ps1 b/images/win/scripts/Installers/Install-Julia.ps1 new file mode 100644 index 000000000..cde01e691 --- /dev/null +++ b/images/win/scripts/Installers/Install-Julia.ps1 @@ -0,0 +1,62 @@ +################################################################################ +## File: Install-Julia.ps1 +## Desc: Install Julia +################################################################################ + +Import-Module -Name ImageHelpers -Force +function Install-JuliaVersion +{ + Param + ( + [String]$juliaVersion, + [Switch]$addToDefaultPath + ) + + # Split versions. + $juliaMajorVersion = $juliaVersion.split(".")[0] + "." + $juliaVersion.split(".")[1] + + # Download the Julia installer. + Write-Host "Downloading Julia $juliaVersion..." + $ProgressPreference = 'SilentlyContinue' + Invoke-WebRequest -UseBasicParsing -Uri "https://julialang-s3.julialang.org/bin/winnt/x64/$juliaMajorVersion/julia-$juliaVersion-win64.exe" -OutFile "julia-$juliaVersion-win64.exe" + + # Install Julia. + # The installer will change in Julia 1.4. For future-proofing, a version check is run. + if ( $juliaMajorVersion -ge "1.4") + { + Start-Process -FilePath "julia-$juliaVersion-win64.exe" -ArgumentList "/SILENT /dir=C:\Julia" -NoNewWindow -Wait + } + else + { + Start-Process -FilePath "julia-$juliaVersion-win64.exe" -ArgumentList "/S /D=C:\Julia" -NoNewWindow -Wait + } + + # Delete unnecessary files to conserve space. + Write-Host "Deleting downloaded installer..." + Remove-Item "julia-$juliaVersion-win64.exe" + + # Make this the default version of Julia? + if ($addToDefaultPath) + { + Write-Host "Adding Julia $juliaVersion to the path..." + # Add the Julia binaries to the path. + Add-MachinePathItem "C:\Julia\Julia-$juliaVersion\bin" | Out-Null + } + + # Done + Write-Host "Done installing Julia $juliaVersion." + return "C:\Julia\Julia-$juliaVersion" +} + +# Install Julia +$juliaVersionsToInstall = $env:JULIA_VERSIONS.split(",") + +foreach($julia in $juliaVersionsToInstall) { + Write-Host "Installing Julia ${julia}" + if($julia -eq $env:JULIA_DEFAULT) { + $installDirectory = Install-JuliaVersion -juliaVersion $julia -addToDefaultPath + } else { + $installDirectory = Install-JuliaVersion -juliaVersion $julia + } + setx $envName "$installDirectory" /M +} diff --git a/images/win/scripts/Installers/Validate-Julia.ps1 b/images/win/scripts/Installers/Validate-Julia.ps1 new file mode 100644 index 000000000..e3c8973cc --- /dev/null +++ b/images/win/scripts/Installers/Validate-Julia.ps1 @@ -0,0 +1,49 @@ +################################################################################ +## File: Validate-Julia.ps1 +## Desc: Validate Julia +################################################################################ + +# Verify that julia.exe is on the path +if(Get-Command -Name 'julia') +{ + Write-Host "$(julia --version) is on the path." +} +else +{ + Write-Host "Julia is not on the path." + exit 1 +} + +# Add details of available versions in Markdown +$tmplMark = @" +#### {0} + +_Environment:_ +* {1}: root directory of the Julia {0} installation + +"@ + +$tmplMarkRoot = @" +#### {0} + +_Environment:_ +* PATH: contains the location of julia.exe version {0} +* {1}: root directory of the Julia {0} installation +"@ + +$SoftwareName = "Julia (x64)" +$Description = New-Object System.Text.StringBuilder +$juliaVersionsToInstall = $env:JULIA_VERSIONS.split(",") + +foreach($julia in $juliaVersionsToInstall) { + $juliaVersion = Get-JuliaVersion + if ($juliaVersion -eq $julia) { + if($julia -eq $env:JULIA_DEFAULT) { + $null = $Description.AppendLine(($tmplMarkRoot -f $juliaVersion)) + } else { + $null = $Description.AppendLine(($tmplMark -f $juliaVersion)) + } + } +} + +Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $Description.ToString() From dd2c76c8a3a1f2d8cde5f4cdfbc40549c2576843 Mon Sep 17 00:00:00 2001 From: Sascha Mann Date: Tue, 11 Feb 2020 13:28:38 +0100 Subject: [PATCH 03/67] Use choco to install Julia --- .../win/scripts/Installers/Install-Julia.ps1 | 58 +------------------ 1 file changed, 1 insertion(+), 57 deletions(-) diff --git a/images/win/scripts/Installers/Install-Julia.ps1 b/images/win/scripts/Installers/Install-Julia.ps1 index cde01e691..0090d5304 100644 --- a/images/win/scripts/Installers/Install-Julia.ps1 +++ b/images/win/scripts/Installers/Install-Julia.ps1 @@ -3,60 +3,4 @@ ## Desc: Install Julia ################################################################################ -Import-Module -Name ImageHelpers -Force -function Install-JuliaVersion -{ - Param - ( - [String]$juliaVersion, - [Switch]$addToDefaultPath - ) - - # Split versions. - $juliaMajorVersion = $juliaVersion.split(".")[0] + "." + $juliaVersion.split(".")[1] - - # Download the Julia installer. - Write-Host "Downloading Julia $juliaVersion..." - $ProgressPreference = 'SilentlyContinue' - Invoke-WebRequest -UseBasicParsing -Uri "https://julialang-s3.julialang.org/bin/winnt/x64/$juliaMajorVersion/julia-$juliaVersion-win64.exe" -OutFile "julia-$juliaVersion-win64.exe" - - # Install Julia. - # The installer will change in Julia 1.4. For future-proofing, a version check is run. - if ( $juliaMajorVersion -ge "1.4") - { - Start-Process -FilePath "julia-$juliaVersion-win64.exe" -ArgumentList "/SILENT /dir=C:\Julia" -NoNewWindow -Wait - } - else - { - Start-Process -FilePath "julia-$juliaVersion-win64.exe" -ArgumentList "/S /D=C:\Julia" -NoNewWindow -Wait - } - - # Delete unnecessary files to conserve space. - Write-Host "Deleting downloaded installer..." - Remove-Item "julia-$juliaVersion-win64.exe" - - # Make this the default version of Julia? - if ($addToDefaultPath) - { - Write-Host "Adding Julia $juliaVersion to the path..." - # Add the Julia binaries to the path. - Add-MachinePathItem "C:\Julia\Julia-$juliaVersion\bin" | Out-Null - } - - # Done - Write-Host "Done installing Julia $juliaVersion." - return "C:\Julia\Julia-$juliaVersion" -} - -# Install Julia -$juliaVersionsToInstall = $env:JULIA_VERSIONS.split(",") - -foreach($julia in $juliaVersionsToInstall) { - Write-Host "Installing Julia ${julia}" - if($julia -eq $env:JULIA_DEFAULT) { - $installDirectory = Install-JuliaVersion -juliaVersion $julia -addToDefaultPath - } else { - $installDirectory = Install-JuliaVersion -juliaVersion $julia - } - setx $envName "$installDirectory" /M -} +choco install julia -y From b6d99efb99160747a93761607c77e57fa1dd3d25 Mon Sep 17 00:00:00 2001 From: Sascha Mann Date: Tue, 11 Feb 2020 13:35:42 +0100 Subject: [PATCH 04/67] Simplify Validate-Julia.ps1 script --- .../win/scripts/Installers/Validate-Julia.ps1 | 36 ++++--------------- 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/images/win/scripts/Installers/Validate-Julia.ps1 b/images/win/scripts/Installers/Validate-Julia.ps1 index e3c8973cc..fa4a26e14 100644 --- a/images/win/scripts/Installers/Validate-Julia.ps1 +++ b/images/win/scripts/Installers/Validate-Julia.ps1 @@ -14,36 +14,12 @@ else exit 1 } -# Add details of available versions in Markdown -$tmplMark = @" -#### {0} - -_Environment:_ -* {1}: root directory of the Julia {0} installation - -"@ - -$tmplMarkRoot = @" -#### {0} - -_Environment:_ -* PATH: contains the location of julia.exe version {0} -* {1}: root directory of the Julia {0} installation -"@ - +# Add description of the software to Markdown $SoftwareName = "Julia (x64)" -$Description = New-Object System.Text.StringBuilder -$juliaVersionsToInstall = $env:JULIA_VERSIONS.split(",") +$juliaVersion = $(julia --version) -match "\d+\.\d+\.\d+" -foreach($julia in $juliaVersionsToInstall) { - $juliaVersion = Get-JuliaVersion - if ($juliaVersion -eq $julia) { - if($julia -eq $env:JULIA_DEFAULT) { - $null = $Description.AppendLine(($tmplMarkRoot -f $juliaVersion)) - } else { - $null = $Description.AppendLine(($tmplMark -f $juliaVersion)) - } - } -} +$Description = @" +_Version:_ $juliaVersion
+"@ -Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $Description.ToString() +Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $Description From be9cfd7f930da345abf37e420c7ffe92fc31cdac Mon Sep 17 00:00:00 2001 From: Sascha Mann Date: Tue, 11 Feb 2020 14:01:38 +0100 Subject: [PATCH 05/67] Fetch latest Julia version from GitHub API --- images/linux/scripts/installers/julia.sh | 36 ++++++++++++++++-------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/images/linux/scripts/installers/julia.sh b/images/linux/scripts/installers/julia.sh index 849ff8f29..eabcc63f0 100644 --- a/images/linux/scripts/installers/julia.sh +++ b/images/linux/scripts/installers/julia.sh @@ -7,21 +7,33 @@ # Source the helpers for use with the script source $HELPER_SCRIPTS/document.sh +# This function fetches the latest Julia release from the GitHub API +# Based on https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c +function GetLatestJuliaRelease () { + curl --silent "https://api.github.com/repos/julialang/julia/releases/latest" | + grep '"tag_name":' | + sed -E 's/.*"([^"]+)".*/\1/' | + sed 's/v//' # remove v prefix +} + + # This function installs Julia using the specified arguments: -# $1=MajorVersion (1.3) -# $2=MajorAndMinorVersion (1.3.1) -# $3=IsDefaultVersion (true or false) +# $1=MajorAndMinorVersion (1.3.1) +# $2=IsDefaultVersion (true or false) function InstallJulia () { - curl -sL "https://julialang-s3.julialang.org/bin/linux/x64/$1/julia-$2-linux-x86_64.tar.gz" -o "julia-$2-linux-x86_64.tar.gz" - mkdir -p "/usr/local/julia$1" - tar -C "/usr/local/julia$1" -xzf "julia-$2-linux-x86_64.tar.gz" --strip-components=1 julia - rm "julia-$2-linux-x86_64.tar.gz" + # Extract Major and Minor version from full version string + juliaMajorAndMinorVersion="$(sed 's/\.*[0-9]//3' <<< $1)" + + curl -sL "https://julialang-s3.julialang.org/bin/linux/x64/$juliaMajorAndMinorVersion/julia-$1-linux-x86_64.tar.gz" -o "julia-$1-linux-x86_64.tar.gz" + mkdir -p "/usr/local/julia$juliaMajorAndMinorVersion" + tar -C "/usr/local/julia$juliaMajorAndMinorVersion" -xzf "julia-$1-linux-x86_64.tar.gz" --strip-components=1 julia + rm "julia-$1-linux-x86_64.tar.gz" # If this version of Julia is to be the default version, # symlink it into the path - if [ "$3" = true ]; then - ln -s "/usr/local/julia$1/bin/julia" /usr/bin/julia + if [ "$2" = true ]; then + ln -s "/usr/local/julia$juliaMajorAndMinorVersion/bin/julia" /usr/bin/julia fi # Run tests to determine that the software installed as expected @@ -29,7 +41,7 @@ function InstallJulia () { # If this version of Julia is to be the default version, # check that it has been added to PATH - if [ "$3" = true ]; then + if [ "$2" = true ]; then if ! command -v julia; then echo "Julia was not installed or found on PATH" exit 1 @@ -37,7 +49,7 @@ function InstallJulia () { fi # Verify output of julia --version - if [ ! "$(/usr/local/julia"$1"/bin/julia --version)" = "julia version $2" ]; then + if [ ! "$(/usr/local/julia"$juliaMajorAndMinorVersion"/bin/julia --version)" = "julia version $1" ]; then echo "Julia was not installed correctly" exit 1 fi @@ -47,4 +59,4 @@ function InstallJulia () { DocumentInstalledItem "Julia ($(julia --version))" } -InstallJulia 1.3 1.3.1 true +InstallJulia "$(GetLatestJuliaRelease)" true From e11f4638dc7f0b1a3d3f8267379240085f06567d Mon Sep 17 00:00:00 2001 From: Sascha Mann Date: Tue, 11 Feb 2020 14:38:33 +0100 Subject: [PATCH 06/67] Fix Major.Minor version regex --- images/linux/scripts/installers/julia.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/linux/scripts/installers/julia.sh b/images/linux/scripts/installers/julia.sh index eabcc63f0..1ca450320 100644 --- a/images/linux/scripts/installers/julia.sh +++ b/images/linux/scripts/installers/julia.sh @@ -23,7 +23,7 @@ function GetLatestJuliaRelease () { function InstallJulia () { # Extract Major and Minor version from full version string - juliaMajorAndMinorVersion="$(sed 's/\.*[0-9]//3' <<< $1)" + juliaMajorAndMinorVersion="$(sed 's/\.[^.]*$//' <<< $1)" curl -sL "https://julialang-s3.julialang.org/bin/linux/x64/$juliaMajorAndMinorVersion/julia-$1-linux-x86_64.tar.gz" -o "julia-$1-linux-x86_64.tar.gz" mkdir -p "/usr/local/julia$juliaMajorAndMinorVersion" From 68fb8ac37ca6a4778ce938bf5d87fd1fbe7b084e Mon Sep 17 00:00:00 2001 From: Sascha Mann Date: Wed, 12 Feb 2020 15:03:26 +0100 Subject: [PATCH 07/67] Fix Julia markdown description --- images/win/scripts/Installers/Validate-Julia.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/win/scripts/Installers/Validate-Julia.ps1 b/images/win/scripts/Installers/Validate-Julia.ps1 index fa4a26e14..e31bd39f6 100644 --- a/images/win/scripts/Installers/Validate-Julia.ps1 +++ b/images/win/scripts/Installers/Validate-Julia.ps1 @@ -16,7 +16,7 @@ else # Add description of the software to Markdown $SoftwareName = "Julia (x64)" -$juliaVersion = $(julia --version) -match "\d+\.\d+\.\d+" +$juliaVersion = $(julia --version).split() -match "\d+\.\d+\.\d+" $Description = @" _Version:_ $juliaVersion
From 036ff7c43c3c2e54050d040d3577b479a899b5bc Mon Sep 17 00:00:00 2001 From: Sascha Mann Date: Wed, 12 Feb 2020 15:17:05 +0100 Subject: [PATCH 08/67] Change Julia install directory --- images/win/scripts/Installers/Install-Julia.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/win/scripts/Installers/Install-Julia.ps1 b/images/win/scripts/Installers/Install-Julia.ps1 index 0090d5304..90547472b 100644 --- a/images/win/scripts/Installers/Install-Julia.ps1 +++ b/images/win/scripts/Installers/Install-Julia.ps1 @@ -3,4 +3,4 @@ ## Desc: Install Julia ################################################################################ -choco install julia -y +choco install julia -y --ia "/D=C:\Julia" From 9b52e4043f96d7d6871cc7903271e328207f3c58 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Fri, 14 Feb 2020 19:09:09 +0300 Subject: [PATCH 09/67] Configure yml files for the images --- .../azure-pipelines/image-generation.yml | 29 +++++++++++++ images.CI/azure-pipelines/ubuntu1604.yml | 22 ++++++++++ images.CI/azure-pipelines/ubuntu1804.yml | 22 ++++++++++ images.CI/azure-pipelines/windows2016.yml | 22 ++++++++++ images.CI/azure-pipelines/windows2019.yml | 22 ++++++++++ images.CI/build-image.ps1 | 41 +++++++++++++++++++ images.CI/cleanup.ps1 | 25 +++++++++++ 7 files changed, 183 insertions(+) create mode 100644 images.CI/azure-pipelines/image-generation.yml create mode 100644 images.CI/azure-pipelines/ubuntu1604.yml create mode 100644 images.CI/azure-pipelines/ubuntu1804.yml create mode 100644 images.CI/azure-pipelines/windows2016.yml create mode 100644 images.CI/azure-pipelines/windows2019.yml create mode 100644 images.CI/build-image.ps1 create mode 100644 images.CI/cleanup.ps1 diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml new file mode 100644 index 000000000..373c75009 --- /dev/null +++ b/images.CI/azure-pipelines/image-generation.yml @@ -0,0 +1,29 @@ +jobs: +- job: + pool: ci-agent-pool + steps: + - script: | + ./images.CI/build-image.ps1 -ResourcesNamePrefix $(Build.BuildNumber) ` + -ClientId $(CLIENT_ID) ` + -ClientSecret $(CLIENT_SECRET) ` + -Image ${{ parameters.image_type }} ` + -ResourceGroup $(AZURE_RESOURCE_GROUP) ` + -StorageAccount $(AZURE_STORAGE_ACCOUNT) ` + -SubscriptionId $(AZURE_SUBSCRIPTION) ` + -TenantId $(AZURE_TENANT) ` + -Location $(AZURE_LOCATION) ` + -VirtualNetworkName $(BUILD_AGENT_VNET_NAME) ` + -VirtualNetworkRG $(BUILD_AGENT_VNET_RESOURCE_GROUP) ` + -VirtualNetworkSubnet $(BUILD_AGENT_SUBNET_NAME) ` + -GitHubFeedToken $(GITHUB_TOKEN) + displayName: Build VM + + - script: | + ./images.CI/cleanup.ps1 -ResourcesNamePrefix $(Build.BuildNumber) ` + -ClientId $(CLIENT_ID) ` + -ClientSecret $(CLIENT_SECRET) ` + -Image ${{ parameters.image_type }} ` + -SubscriptionId $(AZURE_SUBSCRIPTION) ` + -TenantId $(AZURE_TENANT) + displayName: Clean up resources + condition: always() \ No newline at end of file diff --git a/images.CI/azure-pipelines/ubuntu1604.yml b/images.CI/azure-pipelines/ubuntu1604.yml new file mode 100644 index 000000000..c76a39249 --- /dev/null +++ b/images.CI/azure-pipelines/ubuntu1604.yml @@ -0,0 +1,22 @@ +# schedules: +# - cron: "0 0 * * *" +# displayName: Daily +# branches: +# include: +# - master +# always: true + +variables: +- group: "Image Generation Variables" + +trigger: none +pr: + autoCancel: true + branches: + include: + - master + +jobs: +- template: image-generation.yml + parameters: + image_type: ubuntu1604 \ No newline at end of file diff --git a/images.CI/azure-pipelines/ubuntu1804.yml b/images.CI/azure-pipelines/ubuntu1804.yml new file mode 100644 index 000000000..eb7c6dd0a --- /dev/null +++ b/images.CI/azure-pipelines/ubuntu1804.yml @@ -0,0 +1,22 @@ +# schedules: +# - cron: "0 0 * * *" +# displayName: Daily +# branches: +# include: +# - master +# always: true + +variables: +- group: "Image Generation Variables" + +trigger: none +pr: + autoCancel: true + branches: + include: + - master + +jobs: +- template: image-generation.yml + parameters: + image_type: ubuntu1804 \ No newline at end of file diff --git a/images.CI/azure-pipelines/windows2016.yml b/images.CI/azure-pipelines/windows2016.yml new file mode 100644 index 000000000..d3ac23e65 --- /dev/null +++ b/images.CI/azure-pipelines/windows2016.yml @@ -0,0 +1,22 @@ +# schedules: +# - cron: "0 0 * * *" +# displayName: Daily +# branches: +# include: +# - master +# always: true + +variables: +- group: "Image Generation Variables" + +trigger: none +pr: + autoCancel: true + branches: + include: + - master + +jobs: +- template: image-generation.yml + parameters: + image_type: Windows2016-Azure \ No newline at end of file diff --git a/images.CI/azure-pipelines/windows2019.yml b/images.CI/azure-pipelines/windows2019.yml new file mode 100644 index 000000000..93c730033 --- /dev/null +++ b/images.CI/azure-pipelines/windows2019.yml @@ -0,0 +1,22 @@ +# schedules: +# - cron: "0 0 * * *" +# displayName: Daily +# branches: +# include: +# - master +# always: true + +variables: +- group: "Image Generation Variables" + +trigger: none +pr: + autoCancel: true + branches: + include: + - master + +jobs: +- template: image-generation.yml + parameters: + image_type: Windows2019-Azure \ No newline at end of file diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 new file mode 100644 index 000000000..1087abbec --- /dev/null +++ b/images.CI/build-image.ps1 @@ -0,0 +1,41 @@ +param( + [ValidateSet('Windows2019-Azure','Windows2016-Azure','ubuntu1604','ubuntu1804')] + [String] [Parameter (Mandatory=$true)] $Image, + [String] [Parameter (Mandatory=$true)] $ClientId, + [String] [Parameter (Mandatory=$true)] $ClientSecret, + [String] [Parameter (Mandatory=$true)] $GitHubFeedToken, + [String] [Parameter (Mandatory=$true)] $ResourcesNamePrefix, + [String] [Parameter (Mandatory=$true)] $Location, + [String] [Parameter (Mandatory=$true)] $ResourceGroup, + [String] [Parameter (Mandatory=$true)] $StorageAccount, + [String] [Parameter (Mandatory=$true)] $SubscriptionId, + [String] [Parameter (Mandatory=$true)] $TenantId, + [String] [Parameter (Mandatory=$true)] $VirtualNetworkName, + [String] [Parameter (Mandatory=$true)] $VirtualNetworkRG, + [String] [Parameter (Mandatory=$true)] $VirtualNetworkSubnet +) + +$TemplatePath = (Get-ChildItem -Path "images" -Include "$Image.json" -Recurse -Depth 2).FullName +$TempResourceGroupName = "${ResourcesNamePrefix}_${Image}" +$InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper() + +Write-Host "TemplatePath = $TemplatePath" +Write-Host "ResourcesNamePrefix = $ResourcesNamePrefix" +Write-Host "TempResourceGroupName = $TempResourceGroupName" + +Write-Host "Build $Image VM" +packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` + -var "client_id=$ClientId" ` + -var "client_secret=$ClientSecret" ` + -var "install_password=$InstallPassword" ` + -var "github_feed_token=$GitHubFeedToken" ` + -var "location=$Location" ` + -var "resource_group=$ResourceGroup" ` + -var "storage_account=$StorageAccount" ` + -var "subscription_id=$SubscriptionId" ` + -var "temp_resource_group_name=$TempResourceGroupName" ` + -var "tenant_id=$TenantId" ` + -var "virtual_network_name=$VirtualNetworkName" ` + -var "virtual_network_resource_group_name=$VirtualNetworkRG" ` + -var "virtual_network_subnet_name=$VirtualNetworkSubnet" ` + $TemplatePath \ No newline at end of file diff --git a/images.CI/cleanup.ps1 b/images.CI/cleanup.ps1 new file mode 100644 index 000000000..fcf7d946e --- /dev/null +++ b/images.CI/cleanup.ps1 @@ -0,0 +1,25 @@ +param( + [ValidateSet('Windows2019-Azure','Windows2016-Azure','ubuntu1604','ubuntu1804')] + [String] [Parameter (Mandatory=$true)] $Image, + [String] [Parameter (Mandatory=$true)] $ResourcesNamePrefix, + [String] [Parameter (Mandatory=$true)] $ClientId, + [String] [Parameter (Mandatory=$true)] $ClientSecret, + [String] [Parameter (Mandatory=$true)] $SubscriptionId, + [String] [Parameter (Mandatory=$true)] $TenantId +) + +az login --service-principal --username $ClientId --password $ClientSecret --tenant $TenantId | Out-Null + +$TempResourceGroupName = "${ResourcesNamePrefix}_${Image}" + +Write-Host "ResourcesNamePrefix = $ResourcesNamePrefix" +Write-Host "TempResourceGroupName = $TempResourceGroupName" + +$groupExist = az group exists --name $TempResourceGroupName --subscription $SubscriptionId | Out-Null +if ($groupExist -eq "true") { + Write-Host "Found a match, deleting temporary files" + az group delete --name $TempResourceGroupName --subscription $SubscriptionId --yes | Out-Null + Write-Host "Temporary group was deleted succesfully" -ForegroundColor Green +} else { + Write-Host "No temporary groups found" +} \ No newline at end of file From 78ce3a0bd3b903f540b18829e2c83574ddeeafbb Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Fri, 14 Feb 2020 19:44:06 +0300 Subject: [PATCH 10/67] Change the script steps to the PowerShell tasks --- .../azure-pipelines/image-generation.yml | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 373c75009..98922bf0f 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -2,8 +2,12 @@ jobs: - job: pool: ci-agent-pool steps: - - script: | - ./images.CI/build-image.ps1 -ResourcesNamePrefix $(Build.BuildNumber) ` + - task: PowerShell@2 + displayName: 'Build VM' + inputs: + targetType: filePath + filePath: ./images.CI/build-image.ps1 + arguments: -ResourcesNamePrefix $(Build.BuildNumber) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` -Image ${{ parameters.image_type }} ` @@ -16,14 +20,16 @@ jobs: -VirtualNetworkRG $(BUILD_AGENT_VNET_RESOURCE_GROUP) ` -VirtualNetworkSubnet $(BUILD_AGENT_SUBNET_NAME) ` -GitHubFeedToken $(GITHUB_TOKEN) - displayName: Build VM - - script: | - ./images.CI/cleanup.ps1 -ResourcesNamePrefix $(Build.BuildNumber) ` + - task: PowerShell@2 + displayName: 'Clean up resources' + condition: always() + inputs: + targetType: filePath + filePath: ./images.CI/cleanup.ps1 + arguments: -ResourcesNamePrefix $(Build.BuildNumber) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` -Image ${{ parameters.image_type }} ` -SubscriptionId $(AZURE_SUBSCRIPTION) ` - -TenantId $(AZURE_TENANT) - displayName: Clean up resources - condition: always() \ No newline at end of file + -TenantId $(AZURE_TENANT) \ No newline at end of file From 0295b6f57eb5865b228587685b557d19839d7e77 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Sat, 15 Feb 2020 20:52:05 +0300 Subject: [PATCH 11/67] Move variables group --- images.CI/azure-pipelines/image-generation.yml | 4 ++++ images.CI/azure-pipelines/ubuntu1604.yml | 3 --- images.CI/azure-pipelines/ubuntu1804.yml | 3 --- images.CI/azure-pipelines/windows2016.yml | 3 --- images.CI/azure-pipelines/windows2019.yml | 3 --- 5 files changed, 4 insertions(+), 12 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 98922bf0f..2f9840b60 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -1,6 +1,10 @@ jobs: - job: pool: ci-agent-pool + timeoutInMinutes: 600 + variables: + - group: Image Generation Variables + steps: - task: PowerShell@2 displayName: 'Build VM' diff --git a/images.CI/azure-pipelines/ubuntu1604.yml b/images.CI/azure-pipelines/ubuntu1604.yml index c76a39249..b19e79a12 100644 --- a/images.CI/azure-pipelines/ubuntu1604.yml +++ b/images.CI/azure-pipelines/ubuntu1604.yml @@ -6,9 +6,6 @@ # - master # always: true -variables: -- group: "Image Generation Variables" - trigger: none pr: autoCancel: true diff --git a/images.CI/azure-pipelines/ubuntu1804.yml b/images.CI/azure-pipelines/ubuntu1804.yml index eb7c6dd0a..db8acfd9b 100644 --- a/images.CI/azure-pipelines/ubuntu1804.yml +++ b/images.CI/azure-pipelines/ubuntu1804.yml @@ -6,9 +6,6 @@ # - master # always: true -variables: -- group: "Image Generation Variables" - trigger: none pr: autoCancel: true diff --git a/images.CI/azure-pipelines/windows2016.yml b/images.CI/azure-pipelines/windows2016.yml index d3ac23e65..eaac2d1bb 100644 --- a/images.CI/azure-pipelines/windows2016.yml +++ b/images.CI/azure-pipelines/windows2016.yml @@ -6,9 +6,6 @@ # - master # always: true -variables: -- group: "Image Generation Variables" - trigger: none pr: autoCancel: true diff --git a/images.CI/azure-pipelines/windows2019.yml b/images.CI/azure-pipelines/windows2019.yml index 93c730033..ebbf68e7e 100644 --- a/images.CI/azure-pipelines/windows2019.yml +++ b/images.CI/azure-pipelines/windows2019.yml @@ -6,9 +6,6 @@ # - master # always: true -variables: -- group: "Image Generation Variables" - trigger: none pr: autoCancel: true From 624b905bc9e125cc3f38cbea9f53c06b0f028eab Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 17 Feb 2020 13:48:05 +0300 Subject: [PATCH 12/67] Remove log messages --- images.CI/build-image.ps1 | 4 ---- images.CI/cleanup.ps1 | 3 --- 2 files changed, 7 deletions(-) diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 index 1087abbec..989167e44 100644 --- a/images.CI/build-image.ps1 +++ b/images.CI/build-image.ps1 @@ -19,10 +19,6 @@ $TemplatePath = (Get-ChildItem -Path "images" -Include "$Image.json" -Recurse -D $TempResourceGroupName = "${ResourcesNamePrefix}_${Image}" $InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper() -Write-Host "TemplatePath = $TemplatePath" -Write-Host "ResourcesNamePrefix = $ResourcesNamePrefix" -Write-Host "TempResourceGroupName = $TempResourceGroupName" - Write-Host "Build $Image VM" packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` -var "client_id=$ClientId" ` diff --git a/images.CI/cleanup.ps1 b/images.CI/cleanup.ps1 index fcf7d946e..4b0b527be 100644 --- a/images.CI/cleanup.ps1 +++ b/images.CI/cleanup.ps1 @@ -12,9 +12,6 @@ az login --service-principal --username $ClientId --password $ClientSecret --ten $TempResourceGroupName = "${ResourcesNamePrefix}_${Image}" -Write-Host "ResourcesNamePrefix = $ResourcesNamePrefix" -Write-Host "TempResourceGroupName = $TempResourceGroupName" - $groupExist = az group exists --name $TempResourceGroupName --subscription $SubscriptionId | Out-Null if ($groupExist -eq "true") { Write-Host "Found a match, deleting temporary files" From 9b8fe11d2cb0dd83fe3405c445d36bcbb3bd1390 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 17 Feb 2020 17:25:55 +0300 Subject: [PATCH 13/67] Remove ValidateSet --- images.CI/build-image.ps1 | 9 ++++++++- images.CI/cleanup.ps1 | 1 - 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 index 989167e44..9a35373de 100644 --- a/images.CI/build-image.ps1 +++ b/images.CI/build-image.ps1 @@ -1,5 +1,4 @@ param( - [ValidateSet('Windows2019-Azure','Windows2016-Azure','ubuntu1604','ubuntu1804')] [String] [Parameter (Mandatory=$true)] $Image, [String] [Parameter (Mandatory=$true)] $ClientId, [String] [Parameter (Mandatory=$true)] $ClientSecret, @@ -16,9 +15,17 @@ param( ) $TemplatePath = (Get-ChildItem -Path "images" -Include "$Image.json" -Recurse -Depth 2).FullName +if ($TemplatePath -eq $null) +{ + Write-Error "'-Image' parameter is not valid. You have to specify correct image type." + exit 1 +} + $TempResourceGroupName = "${ResourcesNamePrefix}_${Image}" $InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper() +packer validate -syntax-only $TemplatePath + Write-Host "Build $Image VM" packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` -var "client_id=$ClientId" ` diff --git a/images.CI/cleanup.ps1 b/images.CI/cleanup.ps1 index 4b0b527be..00050d551 100644 --- a/images.CI/cleanup.ps1 +++ b/images.CI/cleanup.ps1 @@ -1,5 +1,4 @@ param( - [ValidateSet('Windows2019-Azure','Windows2016-Azure','ubuntu1604','ubuntu1804')] [String] [Parameter (Mandatory=$true)] $Image, [String] [Parameter (Mandatory=$true)] $ResourcesNamePrefix, [String] [Parameter (Mandatory=$true)] $ClientId, From 60dc082e2fc2edc7a70923f258699fc85869b8f2 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Mon, 17 Feb 2020 17:31:08 +0300 Subject: [PATCH 14/67] Minor update --- images.CI/build-image.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 index 9a35373de..6b75ec28f 100644 --- a/images.CI/build-image.ps1 +++ b/images.CI/build-image.ps1 @@ -15,7 +15,7 @@ param( ) $TemplatePath = (Get-ChildItem -Path "images" -Include "$Image.json" -Recurse -Depth 2).FullName -if ($TemplatePath -eq $null) +if (-not $TemplatePath) { Write-Error "'-Image' parameter is not valid. You have to specify correct image type." exit 1 From b42d8129daa7d8b5df9d0b0cf1cda0935b1cb6d4 Mon Sep 17 00:00:00 2001 From: BSKY Date: Mon, 17 Feb 2020 09:01:43 +0900 Subject: [PATCH 15/67] Add zstd --- images/linux/scripts/installers/1604/basic.sh | 6 +++-- images/linux/scripts/installers/1804/basic.sh | 10 +++++--- images/win/Windows2016-Azure.json | 12 ++++++++++ .../win/scripts/Installers/Install-Zstd.ps1 | 6 +++++ .../win/scripts/Installers/Validate-Zstd.ps1 | 24 +++++++++++++++++++ 5 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 images/win/scripts/Installers/Install-Zstd.ps1 create mode 100644 images/win/scripts/Installers/Validate-Zstd.ps1 diff --git a/images/linux/scripts/installers/1604/basic.sh b/images/linux/scripts/installers/1604/basic.sh index fffeba833..99c80a313 100644 --- a/images/linux/scripts/installers/1604/basic.sh +++ b/images/linux/scripts/installers/1604/basic.sh @@ -31,11 +31,12 @@ apt-fast install -y --no-install-recommends \ sudo \ telnet \ time \ + tzdata \ unzip \ upx \ wget \ zip \ - tzdata + zstd # Electron / VSCode / GitHub Desktop prereqs apt-fast install -y --no-install-recommends \ @@ -86,8 +87,9 @@ DocumentInstalledItemIndent "shellcheck" DocumentInstalledItemIndent "sudo" DocumentInstalledItemIndent "telnet" DocumentInstalledItemIndent "time" +DocumentInstalledItemIndent "tzdata" DocumentInstalledItemIndent "unzip" DocumentInstalledItemIndent "upx" DocumentInstalledItemIndent "wget" DocumentInstalledItemIndent "zip" -DocumentInstalledItemIndent "tzdata" +DocumentInstalledItemIndent "zstd" diff --git a/images/linux/scripts/installers/1804/basic.sh b/images/linux/scripts/installers/1804/basic.sh index 043400f8b..c12c7a819 100644 --- a/images/linux/scripts/installers/1804/basic.sh +++ b/images/linux/scripts/installers/1804/basic.sh @@ -58,6 +58,9 @@ apt-get install -y --no-install-recommends telnet echo "Install time" apt-get install -y --no-install-recommends time +echo "Install tzdata" +apt-get install -y --no-install-recommends tzdata + echo "Install unzip" apt-get install -y --no-install-recommends unzip @@ -70,8 +73,8 @@ apt-get install -y --no-install-recommends wget echo "Install zip" apt-get install -y --no-install-recommends zip -echo "Install tzdata" -apt-get install -y --no-install-recommends tzdata +echo "Install zstd" +apt-get install -y --no-install-recommends zstd echo "Install libxkbfile" apt-get install -y --no-install-recommends libxkbfile-dev @@ -155,8 +158,9 @@ DocumentInstalledItemIndent "shellcheck" DocumentInstalledItemIndent "sudo" DocumentInstalledItemIndent "telnet" DocumentInstalledItemIndent "time" +DocumentInstalledItemIndent "tzdata" DocumentInstalledItemIndent "unzip" DocumentInstalledItemIndent "upx" DocumentInstalledItemIndent "wget" DocumentInstalledItemIndent "zip" -DocumentInstalledItemIndent "tzdata" +DocumentInstalledItemIndent "zstd" diff --git a/images/win/Windows2016-Azure.json b/images/win/Windows2016-Azure.json index 98d96feb9..5cba4e2c5 100644 --- a/images/win/Windows2016-Azure.json +++ b/images/win/Windows2016-Azure.json @@ -502,6 +502,12 @@ "{{ template_dir }}/scripts/Installers/Install-Jq.ps1" ] }, + { + "type": "powershell", + "scripts":[ + "{{ template_dir }}/scripts/Installers/Install-Zstd.ps1" + ] + }, { "type": "powershell", "scripts":[ @@ -763,6 +769,12 @@ "{{ template_dir }}/scripts/Installers/Validate-Jq.ps1" ] }, + { + "type": "powershell", + "scripts":[ + "{{ template_dir }}/scripts/Installers/Validate-Zstd.ps1" + ] + }, { "type": "powershell", "scripts":[ diff --git a/images/win/scripts/Installers/Install-Zstd.ps1 b/images/win/scripts/Installers/Install-Zstd.ps1 new file mode 100644 index 000000000..098334ebe --- /dev/null +++ b/images/win/scripts/Installers/Install-Zstd.ps1 @@ -0,0 +1,6 @@ +################################################################################ +## File: Install-zstd.ps1 +## Desc: Install zstd +################################################################################ + +choco install zstandard -y diff --git a/images/win/scripts/Installers/Validate-Zstd.ps1 b/images/win/scripts/Installers/Validate-Zstd.ps1 new file mode 100644 index 000000000..18f3a3343 --- /dev/null +++ b/images/win/scripts/Installers/Validate-Zstd.ps1 @@ -0,0 +1,24 @@ +################################################################################ +## File: Validate-zstd.ps1 +## Desc: Validate zstd +################################################################################ + +if (Get-Command -Name 'zstd') +{ + Write-Host "zstd on path" +} +else +{ + Write-Host 'zstd is not on path' + exit 1 +} + +# Adding description of the software to Markdown +$SoftwareName = "zstd" +$zstdVersion = $(zstd --version).Split(' ')[6].Split(',')[0] + +$Description = @" +_Version:_ $zstdVersion
+"@ + +Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $Description From 6aeaad42ea055ec3720666015c910750e7d454b0 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 18 Feb 2020 15:37:23 +0300 Subject: [PATCH 16/67] Add step with download custom repository --- images.CI/azure-pipelines/image-generation.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 2f9840b60..459b55c37 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -6,6 +6,13 @@ jobs: - group: Image Generation Variables steps: + - pwsh: | + Remove-Item -path './*' -Recurse -Force + Write-Host "Download $(BRANCH_NAME) branch from $(IMAGE_GEN_REPO_URI)" + git clone $(IMAGE_GEN_REPO_URI) . -b $(BRANCH_NAME) --single-branch --depth 1 + displayName: 'Download custom repository' + condition: and(ne(variables['BRANCH_NAME'], ''), ne(variables['IMAGE_GEN_REPO_URI'], '')) + - task: PowerShell@2 displayName: 'Build VM' inputs: From b1a3c37984d814d4a1ecb39b5cf4932c773727dc Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 18 Feb 2020 16:50:11 +0300 Subject: [PATCH 17/67] Add logs filter --- images.CI/build-image.ps1 | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 index 6b75ec28f..dabbb0696 100644 --- a/images.CI/build-image.ps1 +++ b/images.CI/build-image.ps1 @@ -26,6 +26,8 @@ $InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper() packer validate -syntax-only $TemplatePath +$SensitiveData = @('OSType', 'StorageAccountLocation', 'OSDiskUri', 'OSDiskUriReadOnlySas', 'TemplateUri', 'TemplateUriReadOnlySas', ': ->') + Write-Host "Build $Image VM" packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` -var "client_id=$ClientId" ` @@ -41,4 +43,10 @@ packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` -var "virtual_network_name=$VirtualNetworkName" ` -var "virtual_network_resource_group_name=$VirtualNetworkRG" ` -var "virtual_network_subnet_name=$VirtualNetworkSubnet" ` - $TemplatePath \ No newline at end of file + $TemplatePath ` + | Where-Object { + #Filter sensitive data from Packer logs + $currentString = $_ + $matchedString = $SensitiveData | Where-Object { $currentString -match $_ } + return $matchedString -eq $null + } \ No newline at end of file From 9c41221b9ba903c48bb94173e0a9022d5d865b54 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 18 Feb 2020 17:05:00 +0300 Subject: [PATCH 18/67] Minor updates --- images.CI/build-image.ps1 | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 index dabbb0696..73455c021 100644 --- a/images.CI/build-image.ps1 +++ b/images.CI/build-image.ps1 @@ -26,7 +26,13 @@ $InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper() packer validate -syntax-only $TemplatePath -$SensitiveData = @('OSType', 'StorageAccountLocation', 'OSDiskUri', 'OSDiskUriReadOnlySas', 'TemplateUri', 'TemplateUriReadOnlySas', ': ->') +$SensitiveData = @('OSType', + 'StorageAccountLocation', + 'OSDiskUri', + 'OSDiskUriReadOnlySas', + 'TemplateUri', + 'TemplateUriReadOnlySas', + ': ->') Write-Host "Build $Image VM" packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` @@ -47,6 +53,6 @@ packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` | Where-Object { #Filter sensitive data from Packer logs $currentString = $_ - $matchedString = $SensitiveData | Where-Object { $currentString -match $_ } - return $matchedString -eq $null + $sensitiveString = $SensitiveData | Where-Object { $currentString -match $_ } + $sensitiveString -eq $null } \ No newline at end of file From f4bd0f2a6a7b6d72f6570b5ac3a873c4b8de85b8 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 18 Feb 2020 17:09:48 +0300 Subject: [PATCH 19/67] Minor update --- images.CI/build-image.ps1 | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/images.CI/build-image.ps1 b/images.CI/build-image.ps1 index 73455c021..54748e417 100644 --- a/images.CI/build-image.ps1 +++ b/images.CI/build-image.ps1 @@ -26,13 +26,15 @@ $InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper() packer validate -syntax-only $TemplatePath -$SensitiveData = @('OSType', - 'StorageAccountLocation', - 'OSDiskUri', - 'OSDiskUriReadOnlySas', - 'TemplateUri', - 'TemplateUriReadOnlySas', - ': ->') +$SensitiveData = @( + 'OSType', + 'StorageAccountLocation', + 'OSDiskUri', + 'OSDiskUriReadOnlySas', + 'TemplateUri', + 'TemplateUriReadOnlySas', + ': ->' +) Write-Host "Build $Image VM" packer build -var "capture_name_prefix=$ResourcesNamePrefix" ` From 33a765dcc51a12204bbc89906f1217715b838542 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Tue, 18 Feb 2020 17:19:11 +0300 Subject: [PATCH 20/67] Change pwsh step to powershell step --- images.CI/azure-pipelines/image-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 459b55c37..bab1c1c84 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -6,7 +6,7 @@ jobs: - group: Image Generation Variables steps: - - pwsh: | + - powershell: | Remove-Item -path './*' -Recurse -Force Write-Host "Download $(BRANCH_NAME) branch from $(IMAGE_GEN_REPO_URI)" git clone $(IMAGE_GEN_REPO_URI) . -b $(BRANCH_NAME) --single-branch --depth 1 From cc8bdd738467c8803411af1e5571f8ba6b95d2a3 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 19 Feb 2020 09:56:16 +0400 Subject: [PATCH 21/67] dotnetcore-sdk.sh has been refactored for using in both Ubuntu 16/18 deployments --- .vscode/tasks.json | 50 +++------- images/linux/scripts/base/repos.sh | 6 +- .../scripts/installers/1804/dotnetcore-sdk.sh | 99 ------------------- .../installers/{1604 => }/dotnetcore-sdk.sh | 4 + images/linux/ubuntu1604.json | 2 +- images/linux/ubuntu1804.json | 2 +- 6 files changed, 22 insertions(+), 141 deletions(-) delete mode 100644 images/linux/scripts/installers/1804/dotnetcore-sdk.sh rename images/linux/scripts/installers/{1604 => }/dotnetcore-sdk.sh (96%) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index da6002dd0..744b21456 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,42 +1,14 @@ -// Available variables which can be used inside of strings. -// ${workspaceRoot}: the root folder of the team -// ${file}: the current opened file -// ${relativeFile}: the current opened file relative to workspaceRoot -// ${fileBasename}: the current opened file's basename -// ${fileDirname}: the current opened file's dirname -// ${fileExtname}: the current opened file's extension -// ${cwd}: the current working directory of the spawned process { - // See https://go.microsoft.com/fwlink/?LinkId=733558 - // for the documentation about the tasks.json format - "version": "2.0.0", - - // Start PowerShell - "windows": { - "command": "${env:windir}/System32/WindowsPowerShell/v1.0/powershell.exe", - //"command": "${env:ProgramFiles}/PowerShell/6.0.0/powershell.exe", - "args": [ "-NoProfile", "-ExecutionPolicy", "Bypass" ] - }, - "linux": { - "command": "/usr/bin/powershell", - "args": [ "-NoProfile" ] - }, - "osx": { - "command": "/usr/local/bin/powershell", - "args": [ "-NoProfile" ] - }, - - // Associate with test task runner - "tasks": [ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ { - "taskName": "Test", - "suppressTaskName": true, - "isTestCommand": true, - "args": [ - "Write-Host 'Invoking Pester...'; $ProgressPreference = 'SilentlyContinue'; Invoke-Pester -Script test -PesterOption @{IncludeVSCodeMarker=$true};", - "Invoke-Command { Write-Host 'Completed Test task in task runner.' }" - ], - "problemMatcher": "$pester" + "type": "bashdb", + "request": "launch", + "name": "Bash-Debug (simplest configuration)", + "program": "${file}" } - ] -} + ] +} \ No newline at end of file diff --git a/images/linux/scripts/base/repos.sh b/images/linux/scripts/base/repos.sh index f8280cc06..6c56d1cae 100644 --- a/images/linux/scripts/base/repos.sh +++ b/images/linux/scripts/base/repos.sh @@ -9,7 +9,11 @@ LSB_RELEASE=$(lsb_release -rs) # Install Microsoft repository wget https://packages.microsoft.com/config/ubuntu/$LSB_RELEASE/packages-microsoft-prod.deb dpkg -i packages-microsoft-prod.deb -apt-get update +apt-get install apt-transport-https # Install Microsoft GPG public key curl -L https://packages.microsoft.com/keys/microsoft.asc | apt-key add - +curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg +mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg +sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod bionic main" > /etc/apt/sources.list.d/dotnetdev.list' +apt-get update \ No newline at end of file diff --git a/images/linux/scripts/installers/1804/dotnetcore-sdk.sh b/images/linux/scripts/installers/1804/dotnetcore-sdk.sh deleted file mode 100644 index d1fb73ec3..000000000 --- a/images/linux/scripts/installers/1804/dotnetcore-sdk.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash -################################################################################ -## File: dotnetcore-sdk.sh -## Desc: Installs .NET Core SDK -################################################################################ - -source $HELPER_SCRIPTS/apt.sh -source $HELPER_SCRIPTS/document.sh - -LATEST_DOTNET_PACKAGES=("dotnet-sdk-3.0" "dotnet-sdk-3.1") - -LSB_RELEASE=$(lsb_release -rs) - -mksamples() -{ - sdk=$1 - sample=$2 - mkdir "$sdk" - cd "$sdk" || exit - set -e - dotnet help - dotnet new globaljson --sdk-version "$sdk" - dotnet new "$sample" - dotnet restore - dotnet build - set +e - cd .. || exit - rm -rf "$sdk" -} - -set -e - -# Disable telemetry -export DOTNET_CLI_TELEMETRY_OPTOUT=1 - -for latest_package in ${LATEST_DOTNET_PACKAGES[@]}; do - echo "Determing if .NET Core ($latest_package) is installed" - if ! IsInstalled $latest_package; then - echo "Could not find .NET Core ($latest_package), installing..." - curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg - mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg - sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod bionic main" > /etc/apt/sources.list.d/dotnetdev.list' - apt-get install apt-transport-https - apt-get update - apt-get install $latest_package -y - else - echo ".NET Core ($latest_package) is already installed" - fi -done - -# Get list of all released SDKs from channels which are not end-of-life or preview -release_urls=("https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/2.1/releases.json" "https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/2.2/releases.json" "https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/3.0/releases.json" "https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/3.1/releases.json") -sdks=() -for release_url in ${release_urls[@]}; do - echo "${release_url}" - releases=$(curl "${release_url}") - sdks=("${sdks[@]}" $(echo "${releases}" | jq '.releases[]' | jq '.sdk.version')) - sdks=("${sdks[@]}" $(echo "${releases}" | jq '.releases[]' | jq '.sdks[]?' | jq '.version')) -done - -sortedSdks=$(echo ${sdks[@]} | tr ' ' '\n' | grep -v preview | grep -v rc | grep -v display | cut -d\" -f2 | sort -u -r) - -for sdk in $sortedSdks; do - url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$sdk/dotnet-sdk-$sdk-linux-x64.tar.gz" - echo "$url" >> urls - echo "Adding $url to list to download later" -done - -# Download additional SDKs -echo "Downloading release tarballs..." -cat urls | xargs -n 1 -P 16 wget -q -for tarball in *.tar.gz; do - dest="./tmp-$(basename -s .tar.gz $tarball)" - echo "Extracting $tarball to $dest" - mkdir "$dest" && tar -C "$dest" -xzf "$tarball" - rsync -qav "$dest/shared/" /usr/share/dotnet/shared/ - rsync -qav "$dest/host/" /usr/share/dotnet/host/ - rsync -qav "$dest/sdk/" /usr/share/dotnet/sdk/ - rm -rf "$dest" - rm "$tarball" -done -rm urls - -DocumentInstalledItem ".NET Core SDK:" -# Smoke test each SDK -for sdk in $sortedSdks; do - mksamples "$sdk" "console" - mksamples "$sdk" "mstest" - mksamples "$sdk" "xunit" - mksamples "$sdk" "web" - mksamples "$sdk" "mvc" - mksamples "$sdk" "webapi" - DocumentInstalledItemIndent "$sdk" -done - -# NuGetFallbackFolder at /usr/share/dotnet/sdk/NuGetFallbackFolder is warmed up by smoke test -# Additional FTE will just copy to ~/.dotnet/NuGet which provides no benefit on a fungible machine -echo "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1" | tee -a /etc/environment -echo 'export PATH="$PATH:$HOME/.dotnet/tools"' | tee -a /etc/skel/.bashrc diff --git a/images/linux/scripts/installers/1604/dotnetcore-sdk.sh b/images/linux/scripts/installers/dotnetcore-sdk.sh similarity index 96% rename from images/linux/scripts/installers/1604/dotnetcore-sdk.sh rename to images/linux/scripts/installers/dotnetcore-sdk.sh index 7db727558..1d36ef977 100644 --- a/images/linux/scripts/installers/1604/dotnetcore-sdk.sh +++ b/images/linux/scripts/installers/dotnetcore-sdk.sh @@ -4,6 +4,7 @@ ## Desc: Installs .NET Core SDK ################################################################################ +HELPER_SCRIPTS=/repos/virtual-environments/images/linux/scripts/helpers source $HELPER_SCRIPTS/apt.sh source $HELPER_SCRIPTS/document.sh @@ -30,6 +31,9 @@ mksamples() set -e +# Disable telemetry +export DOTNET_CLI_TELEMETRY_OPTOUT=1 + for latest_package in ${LATEST_DOTNET_PACKAGES[@]}; do echo "Determing if .NET Core ($latest_package) is installed" if ! IsInstalled $latest_package; then diff --git a/images/linux/ubuntu1604.json b/images/linux/ubuntu1604.json index f3b95f063..5bd28548c 100644 --- a/images/linux/ubuntu1604.json +++ b/images/linux/ubuntu1604.json @@ -129,7 +129,7 @@ "{{template_dir}}/scripts/installers/docker-compose.sh", "{{template_dir}}/scripts/installers/docker-moby.sh", "{{template_dir}}/scripts/installers/docker.sh", - "{{template_dir}}/scripts/installers/1604/dotnetcore-sdk.sh", + "{{template_dir}}/scripts/installers/dotnetcore-sdk.sh", "{{template_dir}}/scripts/installers/erlang.sh", "{{template_dir}}/scripts/installers/firefox.sh", "{{template_dir}}/scripts/installers/gcc.sh", diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index 5938c2aa5..732de5215 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -132,7 +132,7 @@ "{{template_dir}}/scripts/installers/docker-compose.sh", "{{template_dir}}/scripts/installers/docker-moby.sh", "{{template_dir}}/scripts/installers/docker.sh", - "{{template_dir}}/scripts/installers/1804/dotnetcore-sdk.sh", + "{{template_dir}}/scripts/installers/dotnetcore-sdk.sh", "{{template_dir}}/scripts/installers/erlang.sh", "{{template_dir}}/scripts/installers/firefox.sh", "{{template_dir}}/scripts/installers/gcc.sh", From 02192d4fd6b97da61114896a2d64cdbf67502b50 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 19 Feb 2020 10:11:28 +0400 Subject: [PATCH 22/67] Return to tasks.json from master branch --- .vscode/tasks.json | 48 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 744b21456..7827bd0eb 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,14 +1,42 @@ +// Available variables which can be used inside of strings. +// ${workspaceRoot}: the root folder of the team +// ${file}: the current opened file +// ${relativeFile}: the current opened file relative to workspaceRoot +// ${fileBasename}: the current opened file's basename +// ${fileDirname}: the current opened file's dirname +// ${fileExtname}: the current opened file's extension +// ${cwd}: the current working directory of the spawned process { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + + // Start PowerShell + "windows": { + "command": "${env:windir}/System32/WindowsPowerShell/v1.0/powershell.exe", + //"command": "${env:ProgramFiles}/PowerShell/6.0.0/powershell.exe", + "args": [ "-NoProfile", "-ExecutionPolicy", "Bypass" ] + }, + "linux": { + "command": "/usr/bin/powershell", + "args": [ "-NoProfile" ] + }, + "osx": { + "command": "/usr/local/bin/powershell", + "args": [ "-NoProfile" ] + }, + + // Associate with test task runner + "tasks": [ { - "type": "bashdb", - "request": "launch", - "name": "Bash-Debug (simplest configuration)", - "program": "${file}" + "taskName": "Test", + "suppressTaskName": true, + "isTestCommand": true, + "args": [ + "Write-Host 'Invoking Pester...'; $ProgressPreference = 'SilentlyContinue'; Invoke-Pester -Script test -PesterOption @{IncludeVSCodeMarker=$true};", + "Invoke-Command { Write-Host 'Completed Test task in task runner.' }" + ], + "problemMatcher": "$pester" } - ] + ] } \ No newline at end of file From ae119feb62accd32611e5f1076816aa089188965 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 19 Feb 2020 10:32:44 +0400 Subject: [PATCH 23/67] dotnetcore-sdk.sh - excess debug string has been removed --- images/linux/scripts/installers/dotnetcore-sdk.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/images/linux/scripts/installers/dotnetcore-sdk.sh b/images/linux/scripts/installers/dotnetcore-sdk.sh index 1d36ef977..88b6cb0a2 100644 --- a/images/linux/scripts/installers/dotnetcore-sdk.sh +++ b/images/linux/scripts/installers/dotnetcore-sdk.sh @@ -3,8 +3,6 @@ ## File: dotnetcore-sdk.sh ## Desc: Installs .NET Core SDK ################################################################################ - -HELPER_SCRIPTS=/repos/virtual-environments/images/linux/scripts/helpers source $HELPER_SCRIPTS/apt.sh source $HELPER_SCRIPTS/document.sh From bff4ce772620dd5e77fc493411ac65d1d6e99760 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 19 Feb 2020 11:15:04 +0400 Subject: [PATCH 24/67] Original tasks.json has been reverted --- .vscode/tasks.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 7827bd0eb..da6002dd0 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -39,4 +39,4 @@ "problemMatcher": "$pester" } ] -} \ No newline at end of file +} From 82bdea2dfe53bb63696e63fe5c6b92a0c87d13b6 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Wed, 19 Feb 2020 10:58:59 +0300 Subject: [PATCH 25/67] Update variables names --- images.CI/azure-pipelines/image-generation.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index bab1c1c84..0c05a7e7d 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -8,10 +8,10 @@ jobs: steps: - powershell: | Remove-Item -path './*' -Recurse -Force - Write-Host "Download $(BRANCH_NAME) branch from $(IMAGE_GEN_REPO_URI)" - git clone $(IMAGE_GEN_REPO_URI) . -b $(BRANCH_NAME) --single-branch --depth 1 + Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" + git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 displayName: 'Download custom repository' - condition: and(ne(variables['BRANCH_NAME'], ''), ne(variables['IMAGE_GEN_REPO_URI'], '')) + condition: and(ne(variables['CUSTOM_REPOSITORY_BRANCH'], ''), ne(variables['CUSTOM_REPOSITORY_URL'], '')) - task: PowerShell@2 displayName: 'Build VM' From 68bde6422205d1b7aea26258f3b0376ea7120b01 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Wed, 19 Feb 2020 14:30:44 +0300 Subject: [PATCH 26/67] Change step to PowerShell task --- images.CI/azure-pipelines/image-generation.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 0c05a7e7d..477cdc5f8 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -6,12 +6,15 @@ jobs: - group: Image Generation Variables steps: - - powershell: | - Remove-Item -path './*' -Recurse -Force - Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" - git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 + - task: PowerShell@2 displayName: 'Download custom repository' condition: and(ne(variables['CUSTOM_REPOSITORY_BRANCH'], ''), ne(variables['CUSTOM_REPOSITORY_URL'], '')) + inputs: + targetType: 'inline' + script: | + Remove-Item -path './*' -Recurse -Force + Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" + git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 - task: PowerShell@2 displayName: 'Build VM' From 28a081bda6220bd659bd2334bc521bb7021ee098 Mon Sep 17 00:00:00 2001 From: "Andrey Mishechkin (Akvelon INC)" Date: Thu, 20 Feb 2020 13:49:38 +0400 Subject: [PATCH 27/67] Setting '-y' has been added to installation of apt-transport-https --- images/linux/scripts/base/repos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/linux/scripts/base/repos.sh b/images/linux/scripts/base/repos.sh index 6c56d1cae..abf4586e0 100644 --- a/images/linux/scripts/base/repos.sh +++ b/images/linux/scripts/base/repos.sh @@ -9,7 +9,7 @@ LSB_RELEASE=$(lsb_release -rs) # Install Microsoft repository wget https://packages.microsoft.com/config/ubuntu/$LSB_RELEASE/packages-microsoft-prod.deb dpkg -i packages-microsoft-prod.deb -apt-get install apt-transport-https +apt-get install -y apt-transport-https # Install Microsoft GPG public key curl -L https://packages.microsoft.com/keys/microsoft.asc | apt-key add - From cb18129a4f31957af79b4d3e592cbfdc0e96e6c2 Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Thu, 20 Feb 2020 13:46:38 +0300 Subject: [PATCH 28/67] remove pwsh fix --- images/linux/scripts/installers/1804/powershellcore.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/images/linux/scripts/installers/1804/powershellcore.sh b/images/linux/scripts/installers/1804/powershellcore.sh index 0a5f21081..8ace4742c 100644 --- a/images/linux/scripts/installers/1804/powershellcore.sh +++ b/images/linux/scripts/installers/1804/powershellcore.sh @@ -12,9 +12,6 @@ LSB_RELEASE=$(lsb_release -rs) # Install Powershell apt-get install -y powershell -# Temp fix based on: https://github.com/PowerShell/PowerShell/issues/9746 -sudo apt remove libicu64 - # Run tests to determine that the software installed as expected echo "Testing to make sure that script performed as expected, and basic scenarios work" if ! command -v pwsh; then From a3c8fe11533a3bc4cdc3c45ca145a5d4162858ae Mon Sep 17 00:00:00 2001 From: "Andrey Mishechkin (Akvelon INC)" Date: Thu, 20 Feb 2020 18:26:31 +0400 Subject: [PATCH 29/67] Adding the Ubuntu 18 repository has been revoved from repos.sh --- images/linux/scripts/base/repos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/linux/scripts/base/repos.sh b/images/linux/scripts/base/repos.sh index abf4586e0..847101aea 100644 --- a/images/linux/scripts/base/repos.sh +++ b/images/linux/scripts/base/repos.sh @@ -13,7 +13,7 @@ apt-get install -y apt-transport-https # Install Microsoft GPG public key curl -L https://packages.microsoft.com/keys/microsoft.asc | apt-key add - + curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg -sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod bionic main" > /etc/apt/sources.list.d/dotnetdev.list' apt-get update \ No newline at end of file From 56bc08104e91be4946effd0f070337a49f0dfb6f Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Thu, 20 Feb 2020 17:52:12 +0300 Subject: [PATCH 30/67] Add deployment step (#4) * add release step * add creation release * move file to another folder * resolve comments * minor changes --- .../azure-pipelines/image-generation.yml | 12 +++++++ images.CI/create-release.ps1 | 31 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 images.CI/create-release.ps1 diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 477cdc5f8..eb0136eb5 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -35,6 +35,18 @@ jobs: -VirtualNetworkSubnet $(BUILD_AGENT_SUBNET_NAME) ` -GitHubFeedToken $(GITHUB_TOKEN) + - task: PowerShell@2 + displayName: 'Create release for VM deployment' + inputs: + targetType: filePath + filePath: ./images.CI/create-release.ps1 + arguments: -BuildId $(Build.BuildNumber) ` + -Organization $(RELEASE_TARGET_ORGANIZATION) ` + -DefinitionId $(RELEASE_TARGET_DEFINITION_ID) ` + -Project $(RELEASE_TARGET_PROJECT) ` + -ImageName ${{ parameters.image_type }} ` + -AccessToken $(RELEASE_TARGET_TOKEN) + - task: PowerShell@2 displayName: 'Clean up resources' condition: always() diff --git a/images.CI/create-release.ps1 b/images.CI/create-release.ps1 new file mode 100644 index 000000000..72d0db4e1 --- /dev/null +++ b/images.CI/create-release.ps1 @@ -0,0 +1,31 @@ +param( + [UInt32] [Parameter (Mandatory)] $BuildId, + [String] [Parameter (Mandatory)] $Organization, + [String] [Parameter (Mandatory)] $Project, + [String] [Parameter (Mandatory)] $ImageName, + [String] [Parameter (Mandatory)] $DefinitionId, + [String] [Parameter (Mandatory)] $AccessToken +) + +$Body = @{ + definitionId = $DefinitionId + variables = @{ + ImageBuildId = @{ + value = $BuildId + } + ImageName = @{ + value = $ImageName + } + } + isDraft = "false" +} | ConvertTo-Json -Depth 3 + +$URL = "https://vsrm.dev.azure.com/$Organization/$Project/_apis/release/releases?api-version=5.1" +$base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("'':${AccessToken}")) +$headers = @{ + Authorization = "Basic ${base64AuthInfo}" +} + +$NewRelease = Invoke-RestMethod $URL -Body $Body -Method "POST" -Headers $headers -ContentType "application/json" + +Write-Host "Created release: $($NewRelease._links.web.href)" \ No newline at end of file From 50cd92aade7a77fdfeed8893ff825d7921866804 Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Thu, 20 Feb 2020 19:10:36 +0300 Subject: [PATCH 31/67] Fix github issue --- images.CI/azure-pipelines/image-generation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index eb0136eb5..20adde521 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -14,7 +14,7 @@ jobs: script: | Remove-Item -path './*' -Recurse -Force Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" - git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 + cmd /c "git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 2>&1" - task: PowerShell@2 displayName: 'Build VM' From aa4c025a44514d590048f867353da1a5dc47b47c Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Thu, 20 Feb 2020 22:23:17 +0300 Subject: [PATCH 32/67] Add comment --- images.CI/azure-pipelines/image-generation.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 20adde521..d4fdf0eae 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -14,6 +14,8 @@ jobs: script: | Remove-Item -path './*' -Recurse -Force Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" + # git in powershell sometimes broke the AzDO pipelines with incorrect stdout interpretation + # we redirect the git output to stdout manually to avoid this issue. cmd /c "git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 2>&1" - task: PowerShell@2 From 883c1e3253b5582f3024f02fa89580dbdb4bf9be Mon Sep 17 00:00:00 2001 From: MaksimZhukov Date: Fri, 21 Feb 2020 10:30:54 +0300 Subject: [PATCH 33/67] Update comment --- images.CI/azure-pipelines/image-generation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index d4fdf0eae..518502541 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -14,8 +14,8 @@ jobs: script: | Remove-Item -path './*' -Recurse -Force Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" - # git in powershell sometimes broke the AzDO pipelines with incorrect stdout interpretation - # we redirect the git output to stdout manually to avoid this issue. + # git on self-hosted agent produces some output to stderr even during successful cloning + # use cmd output redirect to overcome it cmd /c "git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 2>&1" - task: PowerShell@2 From 13db647c4d5d3a3bd824e7ad78229be2f654958b Mon Sep 17 00:00:00 2001 From: Ben Gelens Date: Fri, 21 Feb 2020 08:36:31 +0100 Subject: [PATCH 34/67] add az 3.5.0 to images --- images/linux/Ubuntu1604-README.md | 1 + images/linux/Ubuntu1804-README.md | 1 + images/linux/scripts/installers/1604/azpowershell.sh | 7 +++++++ images/linux/scripts/installers/1804/azpowershell.sh | 9 ++++++++- 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/images/linux/Ubuntu1604-README.md b/images/linux/Ubuntu1604-README.md index c74c5cfb7..b025a8f25 100644 --- a/images/linux/Ubuntu1604-README.md +++ b/images/linux/Ubuntu1604-README.md @@ -257,6 +257,7 @@ the - Az Module (2.6.0) - Az Module (2.8.0) - Az Module (3.1.0) +- Az Module (3.5.0) - Cached container images - node:12-alpine (Digest: sha256:0d6db03a05b5d8b417204258cfa34f36eac1c00a54946fb31891e89a642eb449) - node:12 (Digest: sha256:454651174f54836571258a329788574cf6552bddfd1a7113e769bd9fc3776fe6) diff --git a/images/linux/Ubuntu1804-README.md b/images/linux/Ubuntu1804-README.md index b4f030966..7cbcc7a89 100644 --- a/images/linux/Ubuntu1804-README.md +++ b/images/linux/Ubuntu1804-README.md @@ -247,6 +247,7 @@ the - Az Module (2.6.0) - Az Module (2.8.0) - Az Module (3.1.0) +- Az Module (3.5.0) - Cached container images - node:12-alpine (Digest: sha256:0d6db03a05b5d8b417204258cfa34f36eac1c00a54946fb31891e89a642eb449) - node:12 (Digest: sha256:454651174f54836571258a329788574cf6552bddfd1a7113e769bd9fc3776fe6) diff --git a/images/linux/scripts/installers/1604/azpowershell.sh b/images/linux/scripts/installers/1604/azpowershell.sh index 74503d1eb..1e87d9b1e 100644 --- a/images/linux/scripts/installers/1604/azpowershell.sh +++ b/images/linux/scripts/installers/1604/azpowershell.sh @@ -14,6 +14,7 @@ sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_2.3.2 -Requi sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_2.6.0 -RequiredVersion 2.6.0 -Force' sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_2.8.0 -RequiredVersion 2.8.0 -Force' sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_3.1.0 -RequiredVersion 3.1.0 -Force' +sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_3.5.0 -RequiredVersion 3.5.0 -Force' # Run tests to determine that the software installed as expected echo "Testing to make sure that script performed as expected, and basic scenarios work" @@ -46,6 +47,11 @@ if ! pwsh -Command '$actualPSModulePath = $env:PSModulePath ; $env:PSModulePath if (!(get-module -listavailable -name Az.accounts)) { Write-Host "Az Module was not installed"; $env:PSModulePath = $actualPSModulePath; exit 1 } + $env:PSModulePath = $actualPSModulePath + $actualPSModulePath = $env:PSModulePath ; $env:PSModulePath = "/usr/share/az_3.5.0:" + $env:PSModulePath; + if (!(get-module -listavailable -name Az.accounts)) { + Write-Host "Az Module was not installed"; $env:PSModulePath = $actualPSModulePath; exit 1 + } $env:PSModulePath = $actualPSModulePath'; then exit 1 fi @@ -58,3 +64,4 @@ DocumentInstalledItem "Az Module (2.3.2)" DocumentInstalledItem "Az Module (2.6.0)" DocumentInstalledItem "Az Module (2.8.0)" DocumentInstalledItem "Az Module (3.1.0)" +DocumentInstalledItem "Az Module (3.5.0)" diff --git a/images/linux/scripts/installers/1804/azpowershell.sh b/images/linux/scripts/installers/1804/azpowershell.sh index 73d229458..1e87d9b1e 100644 --- a/images/linux/scripts/installers/1804/azpowershell.sh +++ b/images/linux/scripts/installers/1804/azpowershell.sh @@ -14,6 +14,7 @@ sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_2.3.2 -Requi sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_2.6.0 -RequiredVersion 2.6.0 -Force' sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_2.8.0 -RequiredVersion 2.8.0 -Force' sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_3.1.0 -RequiredVersion 3.1.0 -Force' +sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_3.5.0 -RequiredVersion 3.5.0 -Force' # Run tests to determine that the software installed as expected echo "Testing to make sure that script performed as expected, and basic scenarios work" @@ -46,6 +47,11 @@ if ! pwsh -Command '$actualPSModulePath = $env:PSModulePath ; $env:PSModulePath if (!(get-module -listavailable -name Az.accounts)) { Write-Host "Az Module was not installed"; $env:PSModulePath = $actualPSModulePath; exit 1 } + $env:PSModulePath = $actualPSModulePath + $actualPSModulePath = $env:PSModulePath ; $env:PSModulePath = "/usr/share/az_3.5.0:" + $env:PSModulePath; + if (!(get-module -listavailable -name Az.accounts)) { + Write-Host "Az Module was not installed"; $env:PSModulePath = $actualPSModulePath; exit 1 + } $env:PSModulePath = $actualPSModulePath'; then exit 1 fi @@ -57,4 +63,5 @@ DocumentInstalledItem "Az Module (1.6.0)" DocumentInstalledItem "Az Module (2.3.2)" DocumentInstalledItem "Az Module (2.6.0)" DocumentInstalledItem "Az Module (2.8.0)" -DocumentInstalledItem "Az Module (3.1.0)" \ No newline at end of file +DocumentInstalledItem "Az Module (3.1.0)" +DocumentInstalledItem "Az Module (3.5.0)" From 78c3440c50ae517ab754ee4174f8a08940dc7d2a Mon Sep 17 00:00:00 2001 From: Ben Gelens Date: Fri, 21 Feb 2020 08:46:24 +0100 Subject: [PATCH 35/67] reset autogenerated docs --- images/linux/Ubuntu1604-README.md | 1 - images/linux/Ubuntu1804-README.md | 1 - 2 files changed, 2 deletions(-) diff --git a/images/linux/Ubuntu1604-README.md b/images/linux/Ubuntu1604-README.md index b025a8f25..c74c5cfb7 100644 --- a/images/linux/Ubuntu1604-README.md +++ b/images/linux/Ubuntu1604-README.md @@ -257,7 +257,6 @@ the - Az Module (2.6.0) - Az Module (2.8.0) - Az Module (3.1.0) -- Az Module (3.5.0) - Cached container images - node:12-alpine (Digest: sha256:0d6db03a05b5d8b417204258cfa34f36eac1c00a54946fb31891e89a642eb449) - node:12 (Digest: sha256:454651174f54836571258a329788574cf6552bddfd1a7113e769bd9fc3776fe6) diff --git a/images/linux/Ubuntu1804-README.md b/images/linux/Ubuntu1804-README.md index 7cbcc7a89..b4f030966 100644 --- a/images/linux/Ubuntu1804-README.md +++ b/images/linux/Ubuntu1804-README.md @@ -247,7 +247,6 @@ the - Az Module (2.6.0) - Az Module (2.8.0) - Az Module (3.1.0) -- Az Module (3.5.0) - Cached container images - node:12-alpine (Digest: sha256:0d6db03a05b5d8b417204258cfa34f36eac1c00a54946fb31891e89a642eb449) - node:12 (Digest: sha256:454651174f54836571258a329788574cf6552bddfd1a7113e769bd9fc3776fe6) From fb51ff3b1df62287c367ee0f1998aa2a24c52abc Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Fri, 21 Feb 2020 11:34:07 +0300 Subject: [PATCH 36/67] Change image prefix (#14) * add release step * add creation release * move file to another folder * resolve comments * minor changes * change BuildNumber to BuildId --- images.CI/azure-pipelines/image-generation.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 518502541..0e20091f7 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -23,7 +23,7 @@ jobs: inputs: targetType: filePath filePath: ./images.CI/build-image.ps1 - arguments: -ResourcesNamePrefix $(Build.BuildNumber) ` + arguments: -ResourcesNamePrefix $(Build.BuildId) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` -Image ${{ parameters.image_type }} ` @@ -42,7 +42,7 @@ jobs: inputs: targetType: filePath filePath: ./images.CI/create-release.ps1 - arguments: -BuildId $(Build.BuildNumber) ` + arguments: -BuildId $(Build.BuildId) ` -Organization $(RELEASE_TARGET_ORGANIZATION) ` -DefinitionId $(RELEASE_TARGET_DEFINITION_ID) ` -Project $(RELEASE_TARGET_PROJECT) ` @@ -55,7 +55,7 @@ jobs: inputs: targetType: filePath filePath: ./images.CI/cleanup.ps1 - arguments: -ResourcesNamePrefix $(Build.BuildNumber) ` + arguments: -ResourcesNamePrefix $(Build.BuildId) ` -ClientId $(CLIENT_ID) ` -ClientSecret $(CLIENT_SECRET) ` -Image ${{ parameters.image_type }} ` From 6da930087b573b580405c5d6021e3747b40a932e Mon Sep 17 00:00:00 2001 From: root Date: Wed, 19 Feb 2020 09:56:16 +0400 Subject: [PATCH 37/67] dotnetcore-sdk.sh has been refactored for using in both Ubuntu 16/18 deployments --- .vscode/tasks.json | 50 +++------- images/linux/scripts/base/repos.sh | 6 +- .../scripts/installers/1804/dotnetcore-sdk.sh | 99 ------------------- .../installers/{1604 => }/dotnetcore-sdk.sh | 4 + images/linux/ubuntu1604.json | 2 +- images/linux/ubuntu1804.json | 2 +- 6 files changed, 22 insertions(+), 141 deletions(-) delete mode 100644 images/linux/scripts/installers/1804/dotnetcore-sdk.sh rename images/linux/scripts/installers/{1604 => }/dotnetcore-sdk.sh (96%) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index da6002dd0..744b21456 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,42 +1,14 @@ -// Available variables which can be used inside of strings. -// ${workspaceRoot}: the root folder of the team -// ${file}: the current opened file -// ${relativeFile}: the current opened file relative to workspaceRoot -// ${fileBasename}: the current opened file's basename -// ${fileDirname}: the current opened file's dirname -// ${fileExtname}: the current opened file's extension -// ${cwd}: the current working directory of the spawned process { - // See https://go.microsoft.com/fwlink/?LinkId=733558 - // for the documentation about the tasks.json format - "version": "2.0.0", - - // Start PowerShell - "windows": { - "command": "${env:windir}/System32/WindowsPowerShell/v1.0/powershell.exe", - //"command": "${env:ProgramFiles}/PowerShell/6.0.0/powershell.exe", - "args": [ "-NoProfile", "-ExecutionPolicy", "Bypass" ] - }, - "linux": { - "command": "/usr/bin/powershell", - "args": [ "-NoProfile" ] - }, - "osx": { - "command": "/usr/local/bin/powershell", - "args": [ "-NoProfile" ] - }, - - // Associate with test task runner - "tasks": [ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ { - "taskName": "Test", - "suppressTaskName": true, - "isTestCommand": true, - "args": [ - "Write-Host 'Invoking Pester...'; $ProgressPreference = 'SilentlyContinue'; Invoke-Pester -Script test -PesterOption @{IncludeVSCodeMarker=$true};", - "Invoke-Command { Write-Host 'Completed Test task in task runner.' }" - ], - "problemMatcher": "$pester" + "type": "bashdb", + "request": "launch", + "name": "Bash-Debug (simplest configuration)", + "program": "${file}" } - ] -} + ] +} \ No newline at end of file diff --git a/images/linux/scripts/base/repos.sh b/images/linux/scripts/base/repos.sh index f8280cc06..6c56d1cae 100644 --- a/images/linux/scripts/base/repos.sh +++ b/images/linux/scripts/base/repos.sh @@ -9,7 +9,11 @@ LSB_RELEASE=$(lsb_release -rs) # Install Microsoft repository wget https://packages.microsoft.com/config/ubuntu/$LSB_RELEASE/packages-microsoft-prod.deb dpkg -i packages-microsoft-prod.deb -apt-get update +apt-get install apt-transport-https # Install Microsoft GPG public key curl -L https://packages.microsoft.com/keys/microsoft.asc | apt-key add - +curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg +mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg +sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod bionic main" > /etc/apt/sources.list.d/dotnetdev.list' +apt-get update \ No newline at end of file diff --git a/images/linux/scripts/installers/1804/dotnetcore-sdk.sh b/images/linux/scripts/installers/1804/dotnetcore-sdk.sh deleted file mode 100644 index d1fb73ec3..000000000 --- a/images/linux/scripts/installers/1804/dotnetcore-sdk.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash -################################################################################ -## File: dotnetcore-sdk.sh -## Desc: Installs .NET Core SDK -################################################################################ - -source $HELPER_SCRIPTS/apt.sh -source $HELPER_SCRIPTS/document.sh - -LATEST_DOTNET_PACKAGES=("dotnet-sdk-3.0" "dotnet-sdk-3.1") - -LSB_RELEASE=$(lsb_release -rs) - -mksamples() -{ - sdk=$1 - sample=$2 - mkdir "$sdk" - cd "$sdk" || exit - set -e - dotnet help - dotnet new globaljson --sdk-version "$sdk" - dotnet new "$sample" - dotnet restore - dotnet build - set +e - cd .. || exit - rm -rf "$sdk" -} - -set -e - -# Disable telemetry -export DOTNET_CLI_TELEMETRY_OPTOUT=1 - -for latest_package in ${LATEST_DOTNET_PACKAGES[@]}; do - echo "Determing if .NET Core ($latest_package) is installed" - if ! IsInstalled $latest_package; then - echo "Could not find .NET Core ($latest_package), installing..." - curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg - mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg - sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod bionic main" > /etc/apt/sources.list.d/dotnetdev.list' - apt-get install apt-transport-https - apt-get update - apt-get install $latest_package -y - else - echo ".NET Core ($latest_package) is already installed" - fi -done - -# Get list of all released SDKs from channels which are not end-of-life or preview -release_urls=("https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/2.1/releases.json" "https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/2.2/releases.json" "https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/3.0/releases.json" "https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/3.1/releases.json") -sdks=() -for release_url in ${release_urls[@]}; do - echo "${release_url}" - releases=$(curl "${release_url}") - sdks=("${sdks[@]}" $(echo "${releases}" | jq '.releases[]' | jq '.sdk.version')) - sdks=("${sdks[@]}" $(echo "${releases}" | jq '.releases[]' | jq '.sdks[]?' | jq '.version')) -done - -sortedSdks=$(echo ${sdks[@]} | tr ' ' '\n' | grep -v preview | grep -v rc | grep -v display | cut -d\" -f2 | sort -u -r) - -for sdk in $sortedSdks; do - url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$sdk/dotnet-sdk-$sdk-linux-x64.tar.gz" - echo "$url" >> urls - echo "Adding $url to list to download later" -done - -# Download additional SDKs -echo "Downloading release tarballs..." -cat urls | xargs -n 1 -P 16 wget -q -for tarball in *.tar.gz; do - dest="./tmp-$(basename -s .tar.gz $tarball)" - echo "Extracting $tarball to $dest" - mkdir "$dest" && tar -C "$dest" -xzf "$tarball" - rsync -qav "$dest/shared/" /usr/share/dotnet/shared/ - rsync -qav "$dest/host/" /usr/share/dotnet/host/ - rsync -qav "$dest/sdk/" /usr/share/dotnet/sdk/ - rm -rf "$dest" - rm "$tarball" -done -rm urls - -DocumentInstalledItem ".NET Core SDK:" -# Smoke test each SDK -for sdk in $sortedSdks; do - mksamples "$sdk" "console" - mksamples "$sdk" "mstest" - mksamples "$sdk" "xunit" - mksamples "$sdk" "web" - mksamples "$sdk" "mvc" - mksamples "$sdk" "webapi" - DocumentInstalledItemIndent "$sdk" -done - -# NuGetFallbackFolder at /usr/share/dotnet/sdk/NuGetFallbackFolder is warmed up by smoke test -# Additional FTE will just copy to ~/.dotnet/NuGet which provides no benefit on a fungible machine -echo "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1" | tee -a /etc/environment -echo 'export PATH="$PATH:$HOME/.dotnet/tools"' | tee -a /etc/skel/.bashrc diff --git a/images/linux/scripts/installers/1604/dotnetcore-sdk.sh b/images/linux/scripts/installers/dotnetcore-sdk.sh similarity index 96% rename from images/linux/scripts/installers/1604/dotnetcore-sdk.sh rename to images/linux/scripts/installers/dotnetcore-sdk.sh index 7db727558..1d36ef977 100644 --- a/images/linux/scripts/installers/1604/dotnetcore-sdk.sh +++ b/images/linux/scripts/installers/dotnetcore-sdk.sh @@ -4,6 +4,7 @@ ## Desc: Installs .NET Core SDK ################################################################################ +HELPER_SCRIPTS=/repos/virtual-environments/images/linux/scripts/helpers source $HELPER_SCRIPTS/apt.sh source $HELPER_SCRIPTS/document.sh @@ -30,6 +31,9 @@ mksamples() set -e +# Disable telemetry +export DOTNET_CLI_TELEMETRY_OPTOUT=1 + for latest_package in ${LATEST_DOTNET_PACKAGES[@]}; do echo "Determing if .NET Core ($latest_package) is installed" if ! IsInstalled $latest_package; then diff --git a/images/linux/ubuntu1604.json b/images/linux/ubuntu1604.json index ae2ac159e..438cdfded 100644 --- a/images/linux/ubuntu1604.json +++ b/images/linux/ubuntu1604.json @@ -131,7 +131,7 @@ "{{template_dir}}/scripts/installers/docker-compose.sh", "{{template_dir}}/scripts/installers/docker-moby.sh", "{{template_dir}}/scripts/installers/docker.sh", - "{{template_dir}}/scripts/installers/1604/dotnetcore-sdk.sh", + "{{template_dir}}/scripts/installers/dotnetcore-sdk.sh", "{{template_dir}}/scripts/installers/erlang.sh", "{{template_dir}}/scripts/installers/firefox.sh", "{{template_dir}}/scripts/installers/gcc.sh", diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index 3c7ce3279..c9f8cac8a 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -134,7 +134,7 @@ "{{template_dir}}/scripts/installers/docker-compose.sh", "{{template_dir}}/scripts/installers/docker-moby.sh", "{{template_dir}}/scripts/installers/docker.sh", - "{{template_dir}}/scripts/installers/1804/dotnetcore-sdk.sh", + "{{template_dir}}/scripts/installers/dotnetcore-sdk.sh", "{{template_dir}}/scripts/installers/erlang.sh", "{{template_dir}}/scripts/installers/firefox.sh", "{{template_dir}}/scripts/installers/gcc.sh", From aa2a120c58e5418f7294864993a25e9cf67fae28 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 19 Feb 2020 10:11:28 +0400 Subject: [PATCH 38/67] Return to tasks.json from master branch --- .vscode/tasks.json | 48 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 744b21456..7827bd0eb 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,14 +1,42 @@ +// Available variables which can be used inside of strings. +// ${workspaceRoot}: the root folder of the team +// ${file}: the current opened file +// ${relativeFile}: the current opened file relative to workspaceRoot +// ${fileBasename}: the current opened file's basename +// ${fileDirname}: the current opened file's dirname +// ${fileExtname}: the current opened file's extension +// ${cwd}: the current working directory of the spawned process { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + + // Start PowerShell + "windows": { + "command": "${env:windir}/System32/WindowsPowerShell/v1.0/powershell.exe", + //"command": "${env:ProgramFiles}/PowerShell/6.0.0/powershell.exe", + "args": [ "-NoProfile", "-ExecutionPolicy", "Bypass" ] + }, + "linux": { + "command": "/usr/bin/powershell", + "args": [ "-NoProfile" ] + }, + "osx": { + "command": "/usr/local/bin/powershell", + "args": [ "-NoProfile" ] + }, + + // Associate with test task runner + "tasks": [ { - "type": "bashdb", - "request": "launch", - "name": "Bash-Debug (simplest configuration)", - "program": "${file}" + "taskName": "Test", + "suppressTaskName": true, + "isTestCommand": true, + "args": [ + "Write-Host 'Invoking Pester...'; $ProgressPreference = 'SilentlyContinue'; Invoke-Pester -Script test -PesterOption @{IncludeVSCodeMarker=$true};", + "Invoke-Command { Write-Host 'Completed Test task in task runner.' }" + ], + "problemMatcher": "$pester" } - ] + ] } \ No newline at end of file From ce04359372196e2dd7bf7f6b1589723ea69dcdfe Mon Sep 17 00:00:00 2001 From: root Date: Wed, 19 Feb 2020 10:32:44 +0400 Subject: [PATCH 39/67] dotnetcore-sdk.sh - excess debug string has been removed --- images/linux/scripts/installers/dotnetcore-sdk.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/images/linux/scripts/installers/dotnetcore-sdk.sh b/images/linux/scripts/installers/dotnetcore-sdk.sh index 1d36ef977..88b6cb0a2 100644 --- a/images/linux/scripts/installers/dotnetcore-sdk.sh +++ b/images/linux/scripts/installers/dotnetcore-sdk.sh @@ -3,8 +3,6 @@ ## File: dotnetcore-sdk.sh ## Desc: Installs .NET Core SDK ################################################################################ - -HELPER_SCRIPTS=/repos/virtual-environments/images/linux/scripts/helpers source $HELPER_SCRIPTS/apt.sh source $HELPER_SCRIPTS/document.sh From 4851448fc341db84db90ac0b05fe08aaed9bb198 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 19 Feb 2020 11:15:04 +0400 Subject: [PATCH 40/67] Original tasks.json has been reverted --- .vscode/tasks.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 7827bd0eb..da6002dd0 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -39,4 +39,4 @@ "problemMatcher": "$pester" } ] -} \ No newline at end of file +} From fc972505b72176e213e3cc3e86b9b2990d1b4523 Mon Sep 17 00:00:00 2001 From: "Andrey Mishechkin (Akvelon INC)" Date: Thu, 20 Feb 2020 13:49:38 +0400 Subject: [PATCH 41/67] Setting '-y' has been added to installation of apt-transport-https --- images/linux/scripts/base/repos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/linux/scripts/base/repos.sh b/images/linux/scripts/base/repos.sh index 6c56d1cae..abf4586e0 100644 --- a/images/linux/scripts/base/repos.sh +++ b/images/linux/scripts/base/repos.sh @@ -9,7 +9,7 @@ LSB_RELEASE=$(lsb_release -rs) # Install Microsoft repository wget https://packages.microsoft.com/config/ubuntu/$LSB_RELEASE/packages-microsoft-prod.deb dpkg -i packages-microsoft-prod.deb -apt-get install apt-transport-https +apt-get install -y apt-transport-https # Install Microsoft GPG public key curl -L https://packages.microsoft.com/keys/microsoft.asc | apt-key add - From 3755a9ea2480d132a54ade5343a9aa153c851525 Mon Sep 17 00:00:00 2001 From: "Andrey Mishechkin (Akvelon INC)" Date: Thu, 20 Feb 2020 18:26:31 +0400 Subject: [PATCH 42/67] Adding the Ubuntu 18 repository has been revoved from repos.sh --- images/linux/scripts/base/repos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/linux/scripts/base/repos.sh b/images/linux/scripts/base/repos.sh index abf4586e0..847101aea 100644 --- a/images/linux/scripts/base/repos.sh +++ b/images/linux/scripts/base/repos.sh @@ -13,7 +13,7 @@ apt-get install -y apt-transport-https # Install Microsoft GPG public key curl -L https://packages.microsoft.com/keys/microsoft.asc | apt-key add - + curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg -sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod bionic main" > /etc/apt/sources.list.d/dotnetdev.list' apt-get update \ No newline at end of file From d17b6c99164ac9ba2feca552841fdb8078f873dd Mon Sep 17 00:00:00 2001 From: "Andrey Mishechkin (Akvelon INC)" Date: Fri, 21 Feb 2020 13:46:09 +0400 Subject: [PATCH 43/67] 1804/powershellcore.sh - adding the modification of dotnetdev.list file --- images/linux/scripts/installers/1804/powershellcore.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/images/linux/scripts/installers/1804/powershellcore.sh b/images/linux/scripts/installers/1804/powershellcore.sh index 0a5f21081..3a7df62e7 100644 --- a/images/linux/scripts/installers/1804/powershellcore.sh +++ b/images/linux/scripts/installers/1804/powershellcore.sh @@ -8,11 +8,12 @@ source $HELPER_SCRIPTS/document.sh LSB_RELEASE=$(lsb_release -rs) +sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod bionic main" > /etc/apt/sources.list.d/dotnetdev.list' # Install Powershell apt-get install -y powershell -# Temp fix based on: https://github.com/PowerShell/PowerShell/issues/9746 +# Temp fix based on: https://github.com/PowerShell/PowerShell/issues/9746 sudo apt remove libicu64 # Run tests to determine that the software installed as expected From 270945d64bde5a1777562adf8846c29d8bc11f0e Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Fri, 21 Feb 2020 13:40:21 +0300 Subject: [PATCH 44/67] add libicu65 installation --- images/linux/scripts/installers/1804/powershellcore.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/images/linux/scripts/installers/1804/powershellcore.sh b/images/linux/scripts/installers/1804/powershellcore.sh index 8ace4742c..accd923b6 100644 --- a/images/linux/scripts/installers/1804/powershellcore.sh +++ b/images/linux/scripts/installers/1804/powershellcore.sh @@ -8,6 +8,9 @@ source $HELPER_SCRIPTS/document.sh LSB_RELEASE=$(lsb_release -rs) +# libicu64, which comes with php-intl module, has powershell breaking issue https://github.com/PowerShell/PowerShell/issues/9746 +# Fix - install additional libicu65 where the issue is fixed +sudo apt get install libicu65 # Install Powershell apt-get install -y powershell From 934c7ca435fa87d4f518ed0a67ebf1f69cae2b8f Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov Date: Fri, 21 Feb 2020 13:49:25 +0300 Subject: [PATCH 45/67] AZURE_EXTENSION_DIR --- images/linux/scripts/installers/azure-cli.sh | 7 +------ images/linux/scripts/installers/azure-devops-cli.sh | 5 +++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/images/linux/scripts/installers/azure-cli.sh b/images/linux/scripts/installers/azure-cli.sh index 665a188a8..713d23151 100644 --- a/images/linux/scripts/installers/azure-cli.sh +++ b/images/linux/scripts/installers/azure-cli.sh @@ -7,13 +7,8 @@ # Source the helpers for use with the script source $HELPER_SCRIPTS/document.sh -LSB_CODENAME=$(lsb_release -cs) - # Install Azure CLI (instructions taken from https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) -echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $LSB_CODENAME main" | tee /etc/apt/sources.list.d/azure-cli.list -apt-key adv --keyserver packages.microsoft.com --recv-keys B02C46DF417A0893 -apt-get update -apt-get install -y --no-install-recommends apt-transport-https azure-cli +curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash # Run tests to determine that the software installed as expected echo "Testing to make sure that script performed as expected, and basic scenarios work" diff --git a/images/linux/scripts/installers/azure-devops-cli.sh b/images/linux/scripts/installers/azure-devops-cli.sh index 2529c53c2..e709243d6 100644 --- a/images/linux/scripts/installers/azure-devops-cli.sh +++ b/images/linux/scripts/installers/azure-devops-cli.sh @@ -7,6 +7,11 @@ # Source the helpers for use with the script source $HELPER_SCRIPTS/document.sh +# AZURE_EXTENSION_DIR shell variable defines where modules are installed +# https://docs.microsoft.com/en-us/cli/azure/azure-cli-extensions-overview +export AZURE_EXTENSION_DIR=/opt/az/azcliextensions +echo "AZURE_EXTENSION_DIR=$AZURE_EXTENSION_DIR" | tee -a /etc/environment + # install azure devops Cli extension az extension add -n azure-devops From deca0ce1e1cd3f8619876f02aa20f7732f22c720 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 21 Feb 2020 09:24:54 -0500 Subject: [PATCH 46/67] Update Install-DotnetSDK.ps1 --- images/win/scripts/Installers/Install-DotnetSDK.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/images/win/scripts/Installers/Install-DotnetSDK.ps1 b/images/win/scripts/Installers/Install-DotnetSDK.ps1 index f7c9a9e23..5d1545500 100644 --- a/images/win/scripts/Installers/Install-DotnetSDK.ps1 +++ b/images/win/scripts/Installers/Install-DotnetSDK.ps1 @@ -77,7 +77,9 @@ function InstallAllValidSdks() # Remove duplicate entries & preview/rc version from download list # Sort the sdks on version $sdks = @($release.'sdk'); - $sdks += $release.'sdks' | Where-Object { !$_.'version'.Contains('-') -and !$_.'version'.Equals($release.'sdk'.'version') } + + # Remove version 3.1.102 from install list, .NET gave a heads-up that this might cause issues and they are working on a fix. + $sdks += $release.'sdks' | Where-Object { !$_.'version'.Equals('3.1.102') -and !$_.'version'.Contains('-') -and !$_.'version'.Equals($release.'sdk'.'version') } $sdks = $sdks | Sort-Object { [Version] $_.'version' } ForEach ($sdk in $sdks) From 15568d893acceec1b361be4dc1fd0fd6abb998fe Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Sat, 22 Feb 2020 14:55:33 +0300 Subject: [PATCH 47/67] add echo --- images/linux/scripts/installers/1804/powershellcore.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/images/linux/scripts/installers/1804/powershellcore.sh b/images/linux/scripts/installers/1804/powershellcore.sh index accd923b6..61bbb18d0 100644 --- a/images/linux/scripts/installers/1804/powershellcore.sh +++ b/images/linux/scripts/installers/1804/powershellcore.sh @@ -10,6 +10,7 @@ source $HELPER_SCRIPTS/document.sh LSB_RELEASE=$(lsb_release -rs) # libicu64, which comes with php-intl module, has powershell breaking issue https://github.com/PowerShell/PowerShell/issues/9746 # Fix - install additional libicu65 where the issue is fixed +echo "install libicu65" sudo apt get install libicu65 # Install Powershell From 72040db7ae68f3a5814be60e708a57246e1b22d7 Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Sat, 22 Feb 2020 14:57:14 +0300 Subject: [PATCH 48/67] remove sudo --- images/linux/scripts/installers/1804/powershellcore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/linux/scripts/installers/1804/powershellcore.sh b/images/linux/scripts/installers/1804/powershellcore.sh index 61bbb18d0..72ccdfa90 100644 --- a/images/linux/scripts/installers/1804/powershellcore.sh +++ b/images/linux/scripts/installers/1804/powershellcore.sh @@ -11,7 +11,7 @@ LSB_RELEASE=$(lsb_release -rs) # libicu64, which comes with php-intl module, has powershell breaking issue https://github.com/PowerShell/PowerShell/issues/9746 # Fix - install additional libicu65 where the issue is fixed echo "install libicu65" -sudo apt get install libicu65 +apt get install libicu65 # Install Powershell apt-get install -y powershell From b2a3a7f4cc592e766fa7318d1f6f2c6a4be2db1b Mon Sep 17 00:00:00 2001 From: "Andrey Mishechkin (Akvelon INC)" Date: Sun, 23 Feb 2020 12:07:38 +0400 Subject: [PATCH 49/67] Debugging the Ubuntu18 image buld --- images/linux/scripts/installers/1804/powershellcore.sh | 3 +++ images/linux/ubuntu1804.json | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/images/linux/scripts/installers/1804/powershellcore.sh b/images/linux/scripts/installers/1804/powershellcore.sh index 3a7df62e7..9d031949d 100644 --- a/images/linux/scripts/installers/1804/powershellcore.sh +++ b/images/linux/scripts/installers/1804/powershellcore.sh @@ -8,12 +8,15 @@ source $HELPER_SCRIPTS/document.sh LSB_RELEASE=$(lsb_release -rs) +echo "Addin the repository [microsoft-ubuntu-bionic-prod] to /etc/apt/sources.list.d/dotnetdev.list" sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod bionic main" > /etc/apt/sources.list.d/dotnetdev.list' # Install Powershell +echo "Running the PowerShell core installation" apt-get install -y powershell # Temp fix based on: https://github.com/PowerShell/PowerShell/issues/9746 +echo "Remove the libicu64" sudo apt remove libicu64 # Run tests to determine that the software installed as expected diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index c9f8cac8a..8d2a7f49a 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -121,6 +121,8 @@ { "type": "shell", "scripts": [ + "{{template_dir}}/scripts/installers/dotnetcore-sdk.sh", + "{{template_dir}}/scripts/installers/1804/powershellcore.sh", "{{template_dir}}/scripts/installers/7-zip.sh", "{{template_dir}}/scripts/installers/ansible.sh", "{{template_dir}}/scripts/installers/azcopy.sh", @@ -134,7 +136,6 @@ "{{template_dir}}/scripts/installers/docker-compose.sh", "{{template_dir}}/scripts/installers/docker-moby.sh", "{{template_dir}}/scripts/installers/docker.sh", - "{{template_dir}}/scripts/installers/dotnetcore-sdk.sh", "{{template_dir}}/scripts/installers/erlang.sh", "{{template_dir}}/scripts/installers/firefox.sh", "{{template_dir}}/scripts/installers/gcc.sh", @@ -160,7 +161,6 @@ "{{template_dir}}/scripts/installers/1804/php.sh", "{{template_dir}}/scripts/installers/pollinate.sh", "{{template_dir}}/scripts/installers/postgresql.sh", - "{{template_dir}}/scripts/installers/1804/powershellcore.sh", "{{template_dir}}/scripts/installers/ruby.sh", "{{template_dir}}/scripts/installers/rust.sh", "{{template_dir}}/scripts/installers/sbt.sh", From 273c361a641181a205c45d397dcaded827ff0763 Mon Sep 17 00:00:00 2001 From: "Andrey Mishechkin (Akvelon INC)" Date: Sun, 23 Feb 2020 12:57:38 +0400 Subject: [PATCH 50/67] Debug Ubuntu1804 build --- images/linux/ubuntu1804.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index 8d2a7f49a..3e605b32e 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -121,14 +121,14 @@ { "type": "shell", "scripts": [ - "{{template_dir}}/scripts/installers/dotnetcore-sdk.sh", - "{{template_dir}}/scripts/installers/1804/powershellcore.sh", "{{template_dir}}/scripts/installers/7-zip.sh", "{{template_dir}}/scripts/installers/ansible.sh", "{{template_dir}}/scripts/installers/azcopy.sh", "{{template_dir}}/scripts/installers/azure-cli.sh", "{{template_dir}}/scripts/installers/azure-devops-cli.sh", "{{template_dir}}/scripts/installers/1804/basic.sh", + "{{template_dir}}/scripts/installers/dotnetcore-sdk.sh", + "{{template_dir}}/scripts/installers/1804/powershellcore.sh", "{{template_dir}}/scripts/installers/aws.sh", "{{template_dir}}/scripts/installers/build-essential.sh", "{{template_dir}}/scripts/installers/clang.sh", From e586b4ecc2a03cb90762e6085c87790df9da0430 Mon Sep 17 00:00:00 2001 From: "Andrey Mishechkin (Akvelon INC)" Date: Sun, 23 Feb 2020 19:13:39 +0400 Subject: [PATCH 51/67] Debub the Unbuntu builds --- images/linux/scripts/base/repos.sh | 2 +- images/linux/scripts/installers/1604/powershellcore.sh | 1 + images/linux/scripts/installers/docker-moby.sh | 4 ---- images/linux/scripts/installers/docker.sh | 2 -- images/linux/ubuntu1804.json | 4 ++-- 5 files changed, 4 insertions(+), 9 deletions(-) diff --git a/images/linux/scripts/base/repos.sh b/images/linux/scripts/base/repos.sh index 847101aea..4ecd2028c 100644 --- a/images/linux/scripts/base/repos.sh +++ b/images/linux/scripts/base/repos.sh @@ -9,7 +9,7 @@ LSB_RELEASE=$(lsb_release -rs) # Install Microsoft repository wget https://packages.microsoft.com/config/ubuntu/$LSB_RELEASE/packages-microsoft-prod.deb dpkg -i packages-microsoft-prod.deb -apt-get install -y apt-transport-https +apt-get install -y apt-transport-https ca-certificates curl software-properties-common # Install Microsoft GPG public key curl -L https://packages.microsoft.com/keys/microsoft.asc | apt-key add - diff --git a/images/linux/scripts/installers/1604/powershellcore.sh b/images/linux/scripts/installers/1604/powershellcore.sh index 8ace4742c..726faa0bd 100644 --- a/images/linux/scripts/installers/1604/powershellcore.sh +++ b/images/linux/scripts/installers/1604/powershellcore.sh @@ -8,6 +8,7 @@ source $HELPER_SCRIPTS/document.sh LSB_RELEASE=$(lsb_release -rs) +curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /etc/apt/sources.list.d/microsoft-prod.list # Install Powershell apt-get install -y powershell diff --git a/images/linux/scripts/installers/docker-moby.sh b/images/linux/scripts/installers/docker-moby.sh index 625c93cfa..f84e495f1 100644 --- a/images/linux/scripts/installers/docker-moby.sh +++ b/images/linux/scripts/installers/docker-moby.sh @@ -15,10 +15,6 @@ if ! IsInstalled $docker_package; then echo "Docker ($docker_package) was not found. Installing..." apt-get remove -y moby-engine moby-cli apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common - curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - - curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /etc/apt/sources.list.d/microsoft-prod.list - apt-get update apt-get install -y moby-engine moby-cli else echo "Docker ($docker_package) is already installed" diff --git a/images/linux/scripts/installers/docker.sh b/images/linux/scripts/installers/docker.sh index b7fb84e6a..e96e5ab17 100644 --- a/images/linux/scripts/installers/docker.sh +++ b/images/linux/scripts/installers/docker.sh @@ -12,8 +12,6 @@ DOCKER_PACKAGE=moby apt-get remove -y moby-engine mobi-cli apt-get update -apt-get install -y apt-transport-https ca-certificates curl software-properties-common -apt-get update apt-get install -y moby-engine mobi-cli docker pull node:10 diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index 3e605b32e..c9f8cac8a 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -127,8 +127,6 @@ "{{template_dir}}/scripts/installers/azure-cli.sh", "{{template_dir}}/scripts/installers/azure-devops-cli.sh", "{{template_dir}}/scripts/installers/1804/basic.sh", - "{{template_dir}}/scripts/installers/dotnetcore-sdk.sh", - "{{template_dir}}/scripts/installers/1804/powershellcore.sh", "{{template_dir}}/scripts/installers/aws.sh", "{{template_dir}}/scripts/installers/build-essential.sh", "{{template_dir}}/scripts/installers/clang.sh", @@ -136,6 +134,7 @@ "{{template_dir}}/scripts/installers/docker-compose.sh", "{{template_dir}}/scripts/installers/docker-moby.sh", "{{template_dir}}/scripts/installers/docker.sh", + "{{template_dir}}/scripts/installers/dotnetcore-sdk.sh", "{{template_dir}}/scripts/installers/erlang.sh", "{{template_dir}}/scripts/installers/firefox.sh", "{{template_dir}}/scripts/installers/gcc.sh", @@ -161,6 +160,7 @@ "{{template_dir}}/scripts/installers/1804/php.sh", "{{template_dir}}/scripts/installers/pollinate.sh", "{{template_dir}}/scripts/installers/postgresql.sh", + "{{template_dir}}/scripts/installers/1804/powershellcore.sh", "{{template_dir}}/scripts/installers/ruby.sh", "{{template_dir}}/scripts/installers/rust.sh", "{{template_dir}}/scripts/installers/sbt.sh", From eadcb76db553a544c9fa356eec5a82ef82a6d315 Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Mon, 24 Feb 2020 12:49:39 +0300 Subject: [PATCH 52/67] add sqllocaldb to readme --- images/win/scripts/Installers/Validate-DACFx.ps1 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/images/win/scripts/Installers/Validate-DACFx.ps1 b/images/win/scripts/Installers/Validate-DACFx.ps1 index fb815c10e..27b01f5c3 100644 --- a/images/win/scripts/Installers/Validate-DACFx.ps1 +++ b/images/win/scripts/Installers/Validate-DACFx.ps1 @@ -15,6 +15,15 @@ else throw "DACFx is not installed!" } +if(Get-Command -Name 'SqlLocalDB') +{ + $localDbPath = (Get-Command -Name 'SqlLocalDB').Source +} +else +{ + throw "SqlLocalDB is not installed!" +} + function Get-DacFxVersion { $regKey = "HKLM:\SOFTWARE\Microsoft\Microsoft SQL Server\Data-Tier Application Framework\CurrentVersion" @@ -28,6 +37,7 @@ $SoftwareName = "SQL Server Data Tier Application Framework (x64)" $Description = @" _Version:_ $(Get-DacFxVersion)
+* SqlLocalDB is available at $localDbPath "@ Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $Description From d0362fd9576ed9bb7ad186131a55d031c214ae0e Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Mon, 24 Feb 2020 12:56:03 +0300 Subject: [PATCH 53/67] provide more detailed description --- images/win/scripts/Installers/Validate-DACFx.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/win/scripts/Installers/Validate-DACFx.ps1 b/images/win/scripts/Installers/Validate-DACFx.ps1 index 27b01f5c3..c7def3ef4 100644 --- a/images/win/scripts/Installers/Validate-DACFx.ps1 +++ b/images/win/scripts/Installers/Validate-DACFx.ps1 @@ -37,7 +37,7 @@ $SoftwareName = "SQL Server Data Tier Application Framework (x64)" $Description = @" _Version:_ $(Get-DacFxVersion)
-* SqlLocalDB is available at $localDbPath +* SQL Server Express LocalDB is available at $localDbPath "@ Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $Description From e68b9e8b5d47f0a3fe1d4edaf77dc5c656787875 Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Mon, 24 Feb 2020 19:11:13 +0300 Subject: [PATCH 54/67] add condition --- images/linux/scripts/installers/1604/dotnetcore-sdk.sh | 9 ++++++--- images/linux/scripts/installers/1804/dotnetcore-sdk.sh | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/images/linux/scripts/installers/1604/dotnetcore-sdk.sh b/images/linux/scripts/installers/1604/dotnetcore-sdk.sh index 7db727558..0193708e6 100644 --- a/images/linux/scripts/installers/1604/dotnetcore-sdk.sh +++ b/images/linux/scripts/installers/1604/dotnetcore-sdk.sh @@ -53,9 +53,12 @@ done sortedSdks=$(echo ${sdks[@]} | tr ' ' '\n' | grep -v preview | grep -v rc | grep -v display | cut -d\" -f2 | sort -u -r) for sdk in $sortedSdks; do - url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$sdk/dotnet-sdk-$sdk-linux-x64.tar.gz" - echo "$url" >> urls - echo "Adding $url to list to download later" + #temporary avoid 3.1.102 installation due to https://github.com/dotnet/aspnetcore/issues/19133 + if [ $sdk != "3.1.102" ]; then + url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$sdk/dotnet-sdk-$sdk-linux-x64.tar.gz" + echo "$url" >> urls + echo "Adding $url to list to download later" + fi done # Download additional SDKs diff --git a/images/linux/scripts/installers/1804/dotnetcore-sdk.sh b/images/linux/scripts/installers/1804/dotnetcore-sdk.sh index d1fb73ec3..27a922ee8 100644 --- a/images/linux/scripts/installers/1804/dotnetcore-sdk.sh +++ b/images/linux/scripts/installers/1804/dotnetcore-sdk.sh @@ -61,9 +61,12 @@ done sortedSdks=$(echo ${sdks[@]} | tr ' ' '\n' | grep -v preview | grep -v rc | grep -v display | cut -d\" -f2 | sort -u -r) for sdk in $sortedSdks; do - url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$sdk/dotnet-sdk-$sdk-linux-x64.tar.gz" - echo "$url" >> urls - echo "Adding $url to list to download later" + #temporary avoid 3.1.102 installation due to https://github.com/dotnet/aspnetcore/issues/19133 + if [ $sdk != "3.1.102" ]; then + url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$sdk/dotnet-sdk-$sdk-linux-x64.tar.gz" + echo "$url" >> urls + echo "Adding $url to list to download later" + fi done # Download additional SDKs From 07f4ebc3bd93810f120d053e2aac8cc008d88af0 Mon Sep 17 00:00:00 2001 From: Joe Bourne Date: Mon, 24 Feb 2020 14:10:47 -0500 Subject: [PATCH 55/67] Updates to readme and note about mac (#457) --- README.md | 59 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index cf8a7eb08..79c92534b 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,46 @@ # GitHub Actions Virtual Environments -This repository contains the code that we use to create the GitHub Actions [virtual environments](https://help.github.com/en/articles/software-in-virtual-environments-for-github-actions). -Please use the issue templates to submit requests and bug reports related to the installed software. +This repository contains the source used to create the [virtual environments](https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners) for GitHub Actions hosted runners. To file bug reports, or request that tools be added/updated, please [open an issue using the appropriate template](https://github.com/actions/virtual-environments/issues/new/choose). To build a VM machine from this repo's source, see the [instructions](./help/CreateImageAndAzureResources.md). -If you need help with how to set up your workflow file or use a specific tool, -check out the [GitHub Actions Community Forum](https://github.community/t5/GitHub-Actions/bd-p/actions). +For general questions about using the virtual environments or writing your Actions workflow, please open requests in the [GitHub Actions Community Forum](https://github.community/t5/GitHub-Actions/bd-p/actions). -If you need help with how to build VM machine from source code, check out the [documentation](./help/CreateImageAndAzureResources.md). +## Available Environments +| Environment | YAML Label | Included Software | Latest Release & Rollout Progress | +| --------------------|---------------------|--------------------|---------------------| +| Ubuntu 18.04 | `ubuntu-latest` or `ubuntu-18.04` | [ubuntu-18.04] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu18&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu18&redirect=1) +| Ubuntu 16.04 | `ubuntu-16.04` | [ubuntu-16.04] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu16&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu16&redirect=1) | +| macOS 10.15 | `macos-latest` or `macos-10.15` | [macOS-10.15] | *Coming soon* | +| Windows Server 2019 | `windows-latest` or `windows-2019` | [windows-2019] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2019&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2019&redirect=1) +| Windows Server 2016 | `windows-2016` | [windows-2016] | [![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2016&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2016&redirect=1) -## OS's offered -We currently offer Linux, macOS, and Windows virtual environments: +***Looking for other Linux distributions?*** We do not plan to offer other Linux distributions. We recommend using Docker if you'd like to build using other distributions with the hosted virtual environments. Alternatively, you can leverage [self-hosted runners] and fully customize your environment to your needs. -[![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu18&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu18&redirect=1) -[![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu16&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=ubuntu16&redirect=1) -[![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2016&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2016&redirect=1) -[![](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2019&badge=1)](https://actionvirtualenvironmentsstatus.azurewebsites.net/api/status?imageName=windows-2019&redirect=1) +***Where is the macOS source?*** We are in the process of preparing our macOS source to live in this repo so we can take contributions from the community. Until then, we appreciate your patience and ask you continue to make tool requests by filing issues. -- **Linux**. We offer Ubuntu 16.04 and Ubuntu 18.04. We do not plan to offer other Linux distributions. If you want to build using other distributions with the hosted virtual environments, we suggest you use Docker. Alternatively, you can host your own VMs and make use of self-hosted runners. -- **macOS**. We offer macOS Catalina 10.15. -- **Windows**. We offer Windows Server 2016 and Windows Server 2019. +## Software Guidelines +In general, these are the guidelines we consider when deciding what to pre-install: -## Guidelines for what's installed -We follow these rough guidelines when deciding what to pre-install: - -- Tools and ecosystems that are more popular and widely-used will be given priority. -- More recent versions of tools will be given priority over older versions. +- Tools and ecosystems that are broadly popular and widely-used will be given priority. +- Recent versions of tools will be given priority over older versions. - Tools and versions that are deprecated or have reached end-of-life will not be added. -- Tools and versions will be removed 6 months after they are deprecated or have reached end-of-life. +- Tools and versions will typically be removed 6 months after they are deprecated or have reached end-of-life. - If a tool can be installed during the build, we will evaluate how much time is saved -and how much space is used by having the tool pre-installed. + and how much space is used by having the tool pre-installed. -## Updates to the virtual environments -_Cadence_ +## Updates to virtual environments +*Cadence* -You can expect approximately weekly updates to the software on the virtual environments. +We typically deploy weekly updates to the software on the virtual environments. For some tools, we always install the latest at the time of the deployment; for others, we pin the tool to specific version(s). -_Notifications_ +*Following Along / Change Notifications* -Right now, we don't have a great way for you to know when updates to the images are coming. -When we have the code available here, you can watch the Releases for when we generate -candidate environments, and when we deploy new ones. +* **High Impact Changes** (ex. breaking changes, new or deprecated environments) will be posted to the GitHub Changelog on our [blog](https://github.blog/changelog/) and on [twitter](https://twitter.com/GHchangelog). +* **Regular Weekly Rhythm** can be followed by watching [Releases](https://github.com/actions/virtual-environments/releases) to see when we generate candidate environments or deploy new ones. You can also track upcoming changes on the [virtual environment project](https://github.com/actions/virtual-environments/projects/1) to see which issues are under development. + +[ubuntu-18.04]: https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md +[ubuntu-16.04]: https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1604-README.md +[Windows-2019]: https://github.com/actions/virtual-environments/blob/master/images/win/Windows2019-Readme.md +[windows-2016]: https://github.com/actions/virtual-environments/blob/master/images/win/Windows2016-Readme.md +[macOS-10.15]: https://github.com/actions/virtual-environments/blob/master/images/macos/macos-10.15-Readme.md +[self-hosted runners]: https://help.github.com/en/actions/hosting-your-own-runners \ No newline at end of file From b05ca822c29ebb20412aa0ba72fa669852fc40e6 Mon Sep 17 00:00:00 2001 From: Image generation service account Date: Mon, 24 Feb 2020 19:55:47 +0000 Subject: [PATCH 56/67] Updating readme file for AzP.20200217.win16.1 version 20200217 --- images/win/Windows2016-Readme.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/images/win/Windows2016-Readme.md b/images/win/Windows2016-Readme.md index 7bf8c199b..688cd1de7 100644 --- a/images/win/Windows2016-Readme.md +++ b/images/win/Windows2016-Readme.md @@ -1,6 +1,6 @@ # Windows Server 2016 -The following software is installed on machines with the 20200211.1 update. +The following software is installed on machines with the 20200217.1 update. Components marked with **\*** have been upgraded since the previous version of the image. @@ -19,7 +19,7 @@ _Environment:_ ## Docker-compose -_Version:_ 1.25.3
+_Version:_ 1.25.4
_Environment:_ * PATH: contains location of docker-compose.exe @@ -30,9 +30,9 @@ _Version:_ 6.2.4
## Docker images The following container images have been cached: -* mcr.microsoft.com/windows/servercore:ltsc2016 (Digest: sha256:10ea07b6576daae149fe5ec7d0dddc8da673f77d171e2143addeae0b5577a12e) * mcr.microsoft.com/dotnet/framework/aspnet:4.8-windowsservercore-ltsc2016 (Digest: sha256:a1bd1a6dc00f2734b5071b9295f715f36a653e4b2d259c1a4a4d9e8cd6f3ade8) * mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2016 (Digest: sha256:7e81f462be36ba1362062306426c65547e6d63b2eab0b5583808581081393a79) +* mcr.microsoft.com/windows/servercore:ltsc2016 (Digest: sha256:42be24b8810c861cc1b3fe75c5e99f75061cb45fdbae1de46d151c18cc8e6a9a) * microsoft/aspnetcore-build:1.0-2.0 (Digest: sha256:9ecc7c5a8a7a11dca5f08c860165646cb30d084606360a3a72b9cbe447241c0c) * mcr.microsoft.com/windows/nanoserver:10.0.14393.953 (Digest: sha256:fc60bd5ae0e61b334ce1cf1bcbf20c10c36b4c5482a01da319c9c989f9e6e268) @@ -482,12 +482,12 @@ _Environment:_ ## Google Chrome _version:_ -80.0.3987.100 +80.0.3987.106 ## Microsoft Edge _version:_ -80.0.361.50 +80.0.361.54 ## Mozilla Firefox @@ -516,7 +516,7 @@ _Environment:_ #### IE Driver _version:_ -80.0.3987.16 +80.0.3987.106 _Environment:_ * IEWebDriver: location of IEDriverServer.exe @@ -532,7 +532,7 @@ _Environment:_ ## Node.js -_Version:_ 12.15.0
+_Version:_ 12.16.0
_Architecture:_ x64
_Environment:_ * PATH: contains location of node.exe
From cdac96cfe3d2a504008b89e7b8c450dcdaa6f3d3 Mon Sep 17 00:00:00 2001 From: "Andrey Mishechkin (Akvelon INC)" Date: Tue, 25 Feb 2020 10:25:57 +0400 Subject: [PATCH 57/67] powershellcore.sh - Excess 'echo's have been removed --- images/linux/scripts/installers/1804/powershellcore.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/images/linux/scripts/installers/1804/powershellcore.sh b/images/linux/scripts/installers/1804/powershellcore.sh index 9d031949d..3a7df62e7 100644 --- a/images/linux/scripts/installers/1804/powershellcore.sh +++ b/images/linux/scripts/installers/1804/powershellcore.sh @@ -8,15 +8,12 @@ source $HELPER_SCRIPTS/document.sh LSB_RELEASE=$(lsb_release -rs) -echo "Addin the repository [microsoft-ubuntu-bionic-prod] to /etc/apt/sources.list.d/dotnetdev.list" sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod bionic main" > /etc/apt/sources.list.d/dotnetdev.list' # Install Powershell -echo "Running the PowerShell core installation" apt-get install -y powershell # Temp fix based on: https://github.com/PowerShell/PowerShell/issues/9746 -echo "Remove the libicu64" sudo apt remove libicu64 # Run tests to determine that the software installed as expected From 454198d92c03bbec95bf354cb2ded54ff25e9abb Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Tue, 25 Feb 2020 13:50:31 +0300 Subject: [PATCH 58/67] add one more workaround --- images/linux/scripts/installers/1604/dotnetcore-sdk.sh | 7 ++++++- images/linux/scripts/installers/1804/dotnetcore-sdk.sh | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/images/linux/scripts/installers/1604/dotnetcore-sdk.sh b/images/linux/scripts/installers/1604/dotnetcore-sdk.sh index 0193708e6..46f881c59 100644 --- a/images/linux/scripts/installers/1604/dotnetcore-sdk.sh +++ b/images/linux/scripts/installers/1604/dotnetcore-sdk.sh @@ -34,7 +34,12 @@ for latest_package in ${LATEST_DOTNET_PACKAGES[@]}; do echo "Determing if .NET Core ($latest_package) is installed" if ! IsInstalled $latest_package; then echo "Could not find .NET Core ($latest_package), installing..." - apt-get install $latest_package -y + #temporary avoid 3.1.102 installation due to https://github.com/dotnet/aspnetcore/issues/19133 + if [ $latest_package != "dotnet-sdk-3.1" ]; then + apt-get install $latest_package -y + else + apt-get install dotnet-sdk-3.1=3.1.101-1 -y + fi else echo ".NET Core ($latest_package) is already installed" fi diff --git a/images/linux/scripts/installers/1804/dotnetcore-sdk.sh b/images/linux/scripts/installers/1804/dotnetcore-sdk.sh index 27a922ee8..474ad501b 100644 --- a/images/linux/scripts/installers/1804/dotnetcore-sdk.sh +++ b/images/linux/scripts/installers/1804/dotnetcore-sdk.sh @@ -42,7 +42,12 @@ for latest_package in ${LATEST_DOTNET_PACKAGES[@]}; do sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod bionic main" > /etc/apt/sources.list.d/dotnetdev.list' apt-get install apt-transport-https apt-get update - apt-get install $latest_package -y + #temporary avoid 3.1.102 installation due to https://github.com/dotnet/aspnetcore/issues/19133 + if [ $latest_package != "dotnet-sdk-3.1" ]; then + apt-get install $latest_package -y + else + apt-get install dotnet-sdk-3.1=3.1.101-1 -y + fi else echo ".NET Core ($latest_package) is already installed" fi From e9bcc3e9f595d033132b9fe4cfc12ffa5faab5e9 Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Tue, 25 Feb 2020 14:37:54 +0300 Subject: [PATCH 59/67] remove 3.1.102 from sorted sdk list --- .../linux/scripts/installers/1604/dotnetcore-sdk.sh | 12 +++++------- .../linux/scripts/installers/1804/dotnetcore-sdk.sh | 12 +++++------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/images/linux/scripts/installers/1604/dotnetcore-sdk.sh b/images/linux/scripts/installers/1604/dotnetcore-sdk.sh index 46f881c59..c0d48dfdd 100644 --- a/images/linux/scripts/installers/1604/dotnetcore-sdk.sh +++ b/images/linux/scripts/installers/1604/dotnetcore-sdk.sh @@ -55,15 +55,13 @@ for release_url in ${release_urls[@]}; do sdks=("${sdks[@]}" $(echo "${releases}" | jq '.releases[]' | jq '.sdks[]?' | jq '.version')) done -sortedSdks=$(echo ${sdks[@]} | tr ' ' '\n' | grep -v preview | grep -v rc | grep -v display | cut -d\" -f2 | sort -u -r) +#temporary avoid 3.1.102 installation due to https://github.com/dotnet/aspnetcore/issues/19133 +sortedSdks=$(echo ${sdks[@]} | tr ' ' '\n' | grep -v 3.1.102 | grep -v preview | grep -v rc | grep -v display | cut -d\" -f2 | sort -u -r) for sdk in $sortedSdks; do - #temporary avoid 3.1.102 installation due to https://github.com/dotnet/aspnetcore/issues/19133 - if [ $sdk != "3.1.102" ]; then - url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$sdk/dotnet-sdk-$sdk-linux-x64.tar.gz" - echo "$url" >> urls - echo "Adding $url to list to download later" - fi + url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$sdk/dotnet-sdk-$sdk-linux-x64.tar.gz" + echo "$url" >> urls + echo "Adding $url to list to download later" done # Download additional SDKs diff --git a/images/linux/scripts/installers/1804/dotnetcore-sdk.sh b/images/linux/scripts/installers/1804/dotnetcore-sdk.sh index 474ad501b..b2081510d 100644 --- a/images/linux/scripts/installers/1804/dotnetcore-sdk.sh +++ b/images/linux/scripts/installers/1804/dotnetcore-sdk.sh @@ -63,15 +63,13 @@ for release_url in ${release_urls[@]}; do sdks=("${sdks[@]}" $(echo "${releases}" | jq '.releases[]' | jq '.sdks[]?' | jq '.version')) done -sortedSdks=$(echo ${sdks[@]} | tr ' ' '\n' | grep -v preview | grep -v rc | grep -v display | cut -d\" -f2 | sort -u -r) +#temporary avoid 3.1.102 installation due to https://github.com/dotnet/aspnetcore/issues/19133 +sortedSdks=$(echo ${sdks[@]} | tr ' ' '\n' | grep -v 3.1.102 | grep -v preview | grep -v rc | grep -v display | cut -d\" -f2 | sort -u -r) for sdk in $sortedSdks; do - #temporary avoid 3.1.102 installation due to https://github.com/dotnet/aspnetcore/issues/19133 - if [ $sdk != "3.1.102" ]; then - url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$sdk/dotnet-sdk-$sdk-linux-x64.tar.gz" - echo "$url" >> urls - echo "Adding $url to list to download later" - fi + url="https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$sdk/dotnet-sdk-$sdk-linux-x64.tar.gz" + echo "$url" >> urls + echo "Adding $url to list to download later" done # Download additional SDKs From 5ce1c140d195a2cb93737caab597078fbc0936dd Mon Sep 17 00:00:00 2001 From: Mikhail Timofeev Date: Tue, 25 Feb 2020 17:09:24 +0300 Subject: [PATCH 60/67] add another condition to get current releases --- images/win/scripts/Installers/Install-DotnetSDK.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images/win/scripts/Installers/Install-DotnetSDK.ps1 b/images/win/scripts/Installers/Install-DotnetSDK.ps1 index 5d1545500..996887885 100644 --- a/images/win/scripts/Installers/Install-DotnetSDK.ps1 +++ b/images/win/scripts/Installers/Install-DotnetSDK.ps1 @@ -66,7 +66,8 @@ function InstallAllValidSdks() Invoke-WebRequest -Uri $dotnetChannel.'releases.json' -UseBasicParsing -OutFile "releases-$channelVersion.json" $currentReleases = Get-Content -Path "releases-$channelVersion.json" | ConvertFrom-Json # filtering out the preview/rc releases - $currentReleases = $currentReleases.'releases' | Where-Object { !$_.'release-version'.Contains('-') } | Sort-Object { [Version] $_.'release-version' } + # Remove version 3.1.102 from install list, .NET gave a heads-up that this might cause issues and they are working on a fix. https://github.com/dotnet/aspnetcore/issues/19133 + $currentReleases = $currentReleases.'releases' | Where-Object { !$_.'release-version'.Contains('-') -and !$_.'release-version'.Contains('3.1.2') } | Sort-Object { [Version] $_.'release-version' } ForEach ($release in $currentReleases) { if ($release.'sdks'.Count -gt 0) @@ -78,8 +79,7 @@ function InstallAllValidSdks() # Sort the sdks on version $sdks = @($release.'sdk'); - # Remove version 3.1.102 from install list, .NET gave a heads-up that this might cause issues and they are working on a fix. - $sdks += $release.'sdks' | Where-Object { !$_.'version'.Equals('3.1.102') -and !$_.'version'.Contains('-') -and !$_.'version'.Equals($release.'sdk'.'version') } + $sdks += $release.'sdks' | Where-Object { !$_.'version'.Contains('-') -and !$_.'version'.Equals($release.'sdk'.'version') } $sdks = $sdks | Sort-Object { [Version] $_.'version' } ForEach ($sdk in $sdks) From 9cc623aaa951d7d141f399d84a54f0d87b0a8ca7 Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Tue, 25 Feb 2020 19:07:45 +0300 Subject: [PATCH 61/67] Add explanation why we use Azure DevOps CI --- images.CI/azure-pipelines/image-generation.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 0e20091f7..7eeb555f2 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -1,3 +1,9 @@ +# Currently, we use Azure DevOps for Images.CI as a temporary solution until GitHub Actions supports our requirements +# Since we have to use self-hosted machines to run image builds, we need the following features to use GitHub Actions for Images CI: +# - https://github.community/t5/GitHub-Actions/Make-secrets-available-to-builds-of-forks/m-p/30678#M508 +# - https://github.community/t5/GitHub-Actions/GitHub-Actions-Manual-Trigger-Approvals/td-p/31504 +# - https://github.community/t5/GitHub-Actions/Protecting-github-workflows/td-p/30290 + jobs: - job: pool: ci-agent-pool @@ -8,7 +14,7 @@ jobs: steps: - task: PowerShell@2 displayName: 'Download custom repository' - condition: and(ne(variables['CUSTOM_REPOSITORY_BRANCH'], ''), ne(variables['CUSTOM_REPOSITORY_URL'], '')) + condition: and(ne(variables['CUSTOM_REPOSITORY_URL'], ''), ne(variables['CUSTOM_REPOSITORY_BRANCH'], '')) inputs: targetType: 'inline' script: | From f85d7ff2754519d0949bcc991ec47a2e0bfca2cf Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Tue, 25 Feb 2020 19:32:18 +0300 Subject: [PATCH 62/67] enable scheduling --- images.CI/azure-pipelines/ubuntu1604.yml | 14 +++++++------- images.CI/azure-pipelines/ubuntu1804.yml | 14 +++++++------- images.CI/azure-pipelines/windows2016.yml | 14 +++++++------- images.CI/azure-pipelines/windows2019.yml | 14 +++++++------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/images.CI/azure-pipelines/ubuntu1604.yml b/images.CI/azure-pipelines/ubuntu1604.yml index b19e79a12..8f6dcef22 100644 --- a/images.CI/azure-pipelines/ubuntu1604.yml +++ b/images.CI/azure-pipelines/ubuntu1604.yml @@ -1,10 +1,10 @@ -# schedules: -# - cron: "0 0 * * *" -# displayName: Daily -# branches: -# include: -# - master -# always: true +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - master + always: true trigger: none pr: diff --git a/images.CI/azure-pipelines/ubuntu1804.yml b/images.CI/azure-pipelines/ubuntu1804.yml index db8acfd9b..998ba42e6 100644 --- a/images.CI/azure-pipelines/ubuntu1804.yml +++ b/images.CI/azure-pipelines/ubuntu1804.yml @@ -1,10 +1,10 @@ -# schedules: -# - cron: "0 0 * * *" -# displayName: Daily -# branches: -# include: -# - master -# always: true +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - master + always: true trigger: none pr: diff --git a/images.CI/azure-pipelines/windows2016.yml b/images.CI/azure-pipelines/windows2016.yml index eaac2d1bb..667b30bd7 100644 --- a/images.CI/azure-pipelines/windows2016.yml +++ b/images.CI/azure-pipelines/windows2016.yml @@ -1,10 +1,10 @@ -# schedules: -# - cron: "0 0 * * *" -# displayName: Daily -# branches: -# include: -# - master -# always: true +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - master + always: true trigger: none pr: diff --git a/images.CI/azure-pipelines/windows2019.yml b/images.CI/azure-pipelines/windows2019.yml index ebbf68e7e..508fa12b3 100644 --- a/images.CI/azure-pipelines/windows2019.yml +++ b/images.CI/azure-pipelines/windows2019.yml @@ -1,10 +1,10 @@ -# schedules: -# - cron: "0 0 * * *" -# displayName: Daily -# branches: -# include: -# - master -# always: true +schedules: +- cron: "0 0 * * *" + displayName: Daily + branches: + include: + - master + always: true trigger: none pr: From c100b2846dc285534e6827e09cd5b13b989a5ef9 Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Wed, 26 Feb 2020 11:36:04 +0300 Subject: [PATCH 63/67] resolve comments --- images.CI/azure-pipelines/image-generation.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 7eeb555f2..33b29273c 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -1,5 +1,5 @@ -# Currently, we use Azure DevOps for Images.CI as a temporary solution until GitHub Actions supports our requirements -# Since we have to use self-hosted machines to run image builds, we need the following features to use GitHub Actions for Images CI: +# Ideally we would use GitHub Actions for this, but since we use self-hosted machines to run image builds +# we need the following features to use GitHub Actions for Images CI: # - https://github.community/t5/GitHub-Actions/Make-secrets-available-to-builds-of-forks/m-p/30678#M508 # - https://github.community/t5/GitHub-Actions/GitHub-Actions-Manual-Trigger-Approvals/td-p/31504 # - https://github.community/t5/GitHub-Actions/Protecting-github-workflows/td-p/30290 @@ -20,9 +20,12 @@ jobs: script: | Remove-Item -path './*' -Recurse -Force Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" + $env:GIT_REDIRECT_STDERR = '2>&1' + git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 + # git # git on self-hosted agent produces some output to stderr even during successful cloning # use cmd output redirect to overcome it - cmd /c "git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 2>&1" + # cmd /c "git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 2>&1" - task: PowerShell@2 displayName: 'Build VM' From d92fbab6501ed3ef352620267725a9ac9912521c Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Wed, 26 Feb 2020 11:39:53 +0300 Subject: [PATCH 64/67] Update image-generation.yml --- images.CI/azure-pipelines/image-generation.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/images.CI/azure-pipelines/image-generation.yml b/images.CI/azure-pipelines/image-generation.yml index 33b29273c..332f34add 100644 --- a/images.CI/azure-pipelines/image-generation.yml +++ b/images.CI/azure-pipelines/image-generation.yml @@ -18,14 +18,11 @@ jobs: inputs: targetType: 'inline' script: | + Write-Host "Clean up default repository" Remove-Item -path './*' -Recurse -Force Write-Host "Download $(CUSTOM_REPOSITORY_BRANCH) branch from $(CUSTOM_REPOSITORY_URL)" $env:GIT_REDIRECT_STDERR = '2>&1' git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 - # git - # git on self-hosted agent produces some output to stderr even during successful cloning - # use cmd output redirect to overcome it - # cmd /c "git clone $(CUSTOM_REPOSITORY_URL) . -b $(CUSTOM_REPOSITORY_BRANCH) --single-branch --depth 1 2>&1" - task: PowerShell@2 displayName: 'Build VM' From f1d83a3d0927de59724a81bcd1701759939043ac Mon Sep 17 00:00:00 2001 From: Aleksandr Chebotov Date: Wed, 26 Feb 2020 13:04:47 +0300 Subject: [PATCH 65/67] move azpowershell.sh script --- .../scripts/installers/1804/azpowershell.sh | 67 ------------------- .../installers/{1604 => }/azpowershell.sh | 0 images/linux/ubuntu1604.json | 2 +- images/linux/ubuntu1804.json | 2 +- 4 files changed, 2 insertions(+), 69 deletions(-) delete mode 100644 images/linux/scripts/installers/1804/azpowershell.sh rename images/linux/scripts/installers/{1604 => }/azpowershell.sh (100%) diff --git a/images/linux/scripts/installers/1804/azpowershell.sh b/images/linux/scripts/installers/1804/azpowershell.sh deleted file mode 100644 index 1e87d9b1e..000000000 --- a/images/linux/scripts/installers/1804/azpowershell.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -################################################################################ -## File: azpowershell.sh -## Desc: Installed Azure PowerShell -################################################################################ - -# Source the helpers for use with the script -source $HELPER_SCRIPTS/document.sh - -# Install Azure CLI (instructions taken from https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) -sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_1.0.0 -RequiredVersion 1.0.0 -Force' -sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_1.6.0 -RequiredVersion 1.6.0 -Force' -sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_2.3.2 -RequiredVersion 2.3.2 -Force' -sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_2.6.0 -RequiredVersion 2.6.0 -Force' -sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_2.8.0 -RequiredVersion 2.8.0 -Force' -sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_3.1.0 -RequiredVersion 3.1.0 -Force' -sudo pwsh -Command 'Save-Module -Name Az -LiteralPath /usr/share/az_3.5.0 -RequiredVersion 3.5.0 -Force' - -# Run tests to determine that the software installed as expected -echo "Testing to make sure that script performed as expected, and basic scenarios work" -if ! pwsh -Command '$actualPSModulePath = $env:PSModulePath ; $env:PSModulePath = "/usr/share/az_1.0.0:" + $env:PSModulePath; - if (!(get-module -listavailable -name Az.accounts)) { - Write-Host "Az Module was not installed"; $env:PSModulePath = $actualPSModulePath; exit 1 - } - $env:PSModulePath = $actualPSModulePath - $actualPSModulePath = $env:PSModulePath ; $env:PSModulePath = "/usr/share/az_1.6.0:" + $env:PSModulePath; - if (!(get-module -listavailable -name Az.accounts)) { - Write-Host "Az Module was not installed"; $env:PSModulePath = $actualPSModulePath; exit 1 - } - $env:PSModulePath = $actualPSModulePath - $actualPSModulePath = $env:PSModulePath ; $env:PSModulePath = "/usr/share/az_2.3.2:" + $env:PSModulePath; - if (!(get-module -listavailable -name Az.accounts)) { - Write-Host "Az Module was not installed"; $env:PSModulePath = $actualPSModulePath; exit 1 - } - $env:PSModulePath = $actualPSModulePath - $actualPSModulePath = $env:PSModulePath ; $env:PSModulePath = "/usr/share/az_2.6.0:" + $env:PSModulePath; - if (!(get-module -listavailable -name Az.accounts)) { - Write-Host "Az Module was not installed"; $env:PSModulePath = $actualPSModulePath; exit 1 - } - $env:PSModulePath = $actualPSModulePath - $actualPSModulePath = $env:PSModulePath ; $env:PSModulePath = "/usr/share/az_2.8.0:" + $env:PSModulePath; - if (!(get-module -listavailable -name Az.accounts)) { - Write-Host "Az Module was not installed"; $env:PSModulePath = $actualPSModulePath; exit 1 - } - $env:PSModulePath = $actualPSModulePath - $actualPSModulePath = $env:PSModulePath ; $env:PSModulePath = "/usr/share/az_3.1.0:" + $env:PSModulePath; - if (!(get-module -listavailable -name Az.accounts)) { - Write-Host "Az Module was not installed"; $env:PSModulePath = $actualPSModulePath; exit 1 - } - $env:PSModulePath = $actualPSModulePath - $actualPSModulePath = $env:PSModulePath ; $env:PSModulePath = "/usr/share/az_3.5.0:" + $env:PSModulePath; - if (!(get-module -listavailable -name Az.accounts)) { - Write-Host "Az Module was not installed"; $env:PSModulePath = $actualPSModulePath; exit 1 - } - $env:PSModulePath = $actualPSModulePath'; then - exit 1 -fi - -# Document what was added to the image -echo "Lastly, documenting what we added to the metadata file" -DocumentInstalledItem "Az Module (1.0.0)" -DocumentInstalledItem "Az Module (1.6.0)" -DocumentInstalledItem "Az Module (2.3.2)" -DocumentInstalledItem "Az Module (2.6.0)" -DocumentInstalledItem "Az Module (2.8.0)" -DocumentInstalledItem "Az Module (3.1.0)" -DocumentInstalledItem "Az Module (3.5.0)" diff --git a/images/linux/scripts/installers/1604/azpowershell.sh b/images/linux/scripts/installers/azpowershell.sh similarity index 100% rename from images/linux/scripts/installers/1604/azpowershell.sh rename to images/linux/scripts/installers/azpowershell.sh diff --git a/images/linux/ubuntu1604.json b/images/linux/ubuntu1604.json index ae2ac159e..c83f40c3c 100644 --- a/images/linux/ubuntu1604.json +++ b/images/linux/ubuntu1604.json @@ -186,7 +186,7 @@ "type": "shell", "scripts":[ "{{template_dir}}/scripts/installers/1604/android.sh", - "{{template_dir}}/scripts/installers/1604/azpowershell.sh", + "{{template_dir}}/scripts/installers/azpowershell.sh", "{{template_dir}}/scripts/helpers/containercache.sh", "{{template_dir}}/scripts/installers/hosted-tool-cache.sh", "{{template_dir}}/scripts/installers/python.sh", diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index 3c7ce3279..67c833b76 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -188,7 +188,7 @@ "type": "shell", "scripts":[ "{{template_dir}}/scripts/installers/1804/android.sh", - "{{template_dir}}/scripts/installers/1804/azpowershell.sh", + "{{template_dir}}/scripts/installers/azpowershell.sh", "{{template_dir}}/scripts/helpers/containercache.sh", "{{template_dir}}/scripts/installers/hosted-tool-cache.sh", "{{template_dir}}/scripts/installers/python.sh", From aca1c5af1c1e99ca7f4164a20720554ec738b558 Mon Sep 17 00:00:00 2001 From: Andy Mishechkin Date: Wed, 26 Feb 2020 19:08:12 +0400 Subject: [PATCH 66/67] powershellcore.sh has been modified --- images/linux/scripts/installers/1604/powershellcore.sh | 1 - images/linux/scripts/installers/1804/powershellcore.sh | 3 --- 2 files changed, 4 deletions(-) diff --git a/images/linux/scripts/installers/1604/powershellcore.sh b/images/linux/scripts/installers/1604/powershellcore.sh index 726faa0bd..c71ad2c2f 100644 --- a/images/linux/scripts/installers/1604/powershellcore.sh +++ b/images/linux/scripts/installers/1604/powershellcore.sh @@ -7,7 +7,6 @@ # Source the helpers for use with the script source $HELPER_SCRIPTS/document.sh -LSB_RELEASE=$(lsb_release -rs) curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /etc/apt/sources.list.d/microsoft-prod.list # Install Powershell diff --git a/images/linux/scripts/installers/1804/powershellcore.sh b/images/linux/scripts/installers/1804/powershellcore.sh index c85de5a9d..2f1922493 100644 --- a/images/linux/scripts/installers/1804/powershellcore.sh +++ b/images/linux/scripts/installers/1804/powershellcore.sh @@ -7,9 +7,6 @@ # Source the helpers for use with the script source $HELPER_SCRIPTS/document.sh -LSB_RELEASE=$(lsb_release -rs) - -sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod bionic main" > /etc/apt/sources.list.d/dotnetdev.list' # libicu64, which comes with php-intl module, has powershell breaking issue https://github.com/PowerShell/PowerShell/issues/9746 # Fix - install additional libicu65 where the issue is fixed echo "install libicu65" From cb6d440538821c1bfa1876bf193e3f2d9d297dcb Mon Sep 17 00:00:00 2001 From: Andy Mishechkin Date: Wed, 26 Feb 2020 22:40:56 +0400 Subject: [PATCH 67/67] 1604\powershell.core has been modified --- images/linux/scripts/installers/1604/powershellcore.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/images/linux/scripts/installers/1604/powershellcore.sh b/images/linux/scripts/installers/1604/powershellcore.sh index c71ad2c2f..5c615ae31 100644 --- a/images/linux/scripts/installers/1604/powershellcore.sh +++ b/images/linux/scripts/installers/1604/powershellcore.sh @@ -7,8 +7,6 @@ # Source the helpers for use with the script source $HELPER_SCRIPTS/document.sh -curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /etc/apt/sources.list.d/microsoft-prod.list - # Install Powershell apt-get install -y powershell