[Ubuntu] Implement new directories hierarchy (#8627)

.github/workflows/ubuntu2004.yml

@@ -5,7 +5,7 @@ on:
   pull_request_target:
     types: labeled
     paths:
-    - 'images/linux/**'
+    - 'images/ubuntu/**'
 
 defaults:
   run:

.github/workflows/ubuntu2204.yml

@@ -5,7 +5,7 @@ on:
   pull_request_target:
     types: labeled
     paths:
-    - 'images/linux/**'
+    - 'images/ubuntu/**'
 
 defaults:
   run:

CONTRIBUTING.md

@@ -49,11 +49,11 @@ Add `Invoke-PesterTests -TestFile <testFileName> [-TestName <describeName>]` at
 
 ### Ubuntu
 
-- Add script that will install and validate the tool and put the script in the `scripts/installers` folder.
+- Add script that will install and validate the tool and put the script in the `scripts/build` folder.
-Use existing scripts such as [github-cli.sh](images/linux/scripts/installers/github-cli.sh) as a starting point.
+Use existing scripts such as [github-cli.sh](images/ubuntu/scripts/build/github-cli.sh) as a starting point.
-  - Use [helpers](images/linux/scripts/helpers/install.sh) to simplify installation process.
+  - Use [helpers](images/ubuntu/scripts/helpers/install.sh) to simplify installation process.
   - Validation part should `exit 1` if any issue with installation.
-- Add changes to the software report generator `images/linux/scripts/SoftwareReport/SoftwareReport.Generator.ps1`. The software report generator is used to generate an image's README file, e.g. [Ubuntu2004-Readme.md](images/linux/Ubuntu2004-README.md) and it uses [MarkdownPS](https://github.com/Sarafian/MarkdownPS).
+- Add changes to the software report generator `images/ubuntu/scripts/docs-gen/SoftwareReport.Generator.ps1`. The software report generator is used to generate an image's README file, e.g. [Ubuntu2004-Readme.md](images/ubuntu/Ubuntu2004-README.md) and it uses [MarkdownPS](https://github.com/Sarafian/MarkdownPS).
 
 ### macOS
 

README.md

@@ -34,8 +34,8 @@ To build a VM machine from this repo's source, see the [instructions](docs/creat
 - In general the `-latest` label is used for the latest OS image version that is GA
 - Before moving the`-latest` label to a new OS version we will announce the change and give sufficient lead time for users to update their workflows
 
-[ubuntu-22.04]: https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md
+[ubuntu-22.04]: https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md
-[ubuntu-20.04]: https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md
+[ubuntu-20.04]: https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md
 [windows-2022]: https://github.com/actions/runner-images/blob/main/images/windows/Windows2022-Readme.md
 [windows-2019]: https://github.com/actions/runner-images/blob/main/images/windows/Windows2019-Readme.md
 [macOS-11]: https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md

docs/create-image-and-azure-resources.md

@@ -272,7 +272,7 @@ Generated tool versions and details can be found in related projects:
 The user, created during the image generation, does not exist in the result image hence some configuration files related to the user's home directory need to be changed as well as the file permissions for some directories. Scripts for that are located in the `post-gen` folder in the repository:
 
 - Windows: <https://github.com/actions/runner-images/tree/main/images/windows/assets/post-gen>
-- Linux: <https://github.com/actions/runner-images/tree/main/images/linux/post-generation>
+- Linux: <https://github.com/actions/runner-images/tree/main/images/ubuntu/assets/post-gen>
 
 **Note:** The default user for Linux should have `sudo privileges`.
 

helpers/GenerateResourcesAndImage.ps1

@@ -24,13 +24,13 @@ Function Get-PackerTemplatePath {
             $relativeTemplatePath = Join-Path "windows" "templates" "windows-2022.json"
         }
         ([ImageType]::Ubuntu2004) {
-            $relativeTemplatePath = Join-Path "linux" "ubuntu2004.json"
+            $relativeTemplatePath = Join-Path "ubuntu" "templates" "ubuntu-2004.json"
         }
         ([ImageType]::Ubuntu2204) {
-            $relativeTemplatePath = Join-Path "linux" "ubuntu2204.pkr.hcl"
+            $relativeTemplatePath = Join-Path "ubuntu" "templates" "ubuntu-2204.pkr.hcl"
         }
         ([ImageType]::UbuntuMinimal) {
-            $relativeTemplatePath = Join-Path "linux" "ubuntuminimal.pkr.hcl"
+            $relativeTemplatePath = Join-Path "ubuntu" "templates" "ubuntu-minimal.pkr.hcl"
         }
         default { throw "Unknown type of image" }
     }

images.CI/linux-and-win/azure-pipelines/image-generation.yml

@@ -65,9 +65,9 @@ jobs:
       targetType: 'inline'
       script: |
         $ImageType = "${{ parameters.image_type }}"
-        $TemplateDirectoryName = if ($ImageType.StartsWith("ubuntu")) { "linux" } else { "windows/templates" }
+        $TemplateDirectoryName = if ($ImageType.StartsWith("ubuntu")) { "ubuntu/templates" } else { "windows/templates" }
         $TemplateDirectoryPath = Join-Path "images" $TemplateDirectoryName | Resolve-Path
-        
+
         $TemplateFileName = "${{ parameters.image_template_name }}"
         if ($TemplateFileName) {
           $TemplatePath = Join-Path $TemplateDirectoryPath $TemplateFileName

images.CI/linux-and-win/azure-pipelines/ubuntu2004.yml

@@ -17,4 +17,5 @@ jobs:
 - template: image-generation.yml
   parameters:
     image_type: ubuntu2004
-    image_readme_name: Ubuntu2004-Readme.md
+    image_readme_name: Ubuntu2004-Readme.md
+    image_template_name: ubuntu-20.04.json

images.CI/linux-and-win/azure-pipelines/ubuntu2204.yml

@@ -17,4 +17,5 @@ jobs:
 - template: image-generation.yml
   parameters:
     image_type: ubuntu2204
-    image_readme_name: Ubuntu2204-Readme.md
+    image_readme_name: Ubuntu2204-Readme.md
+    image_template_name: ubuntu-22.04.pkr.hcl

images.CI/shebang-linter.ps1

@@ -22,7 +22,7 @@ function Validate-Scripts {
     return $ScriptWithoutShebangLine
 }
 
-$PathUbuntu = "./images/linux/scripts"
+$PathUbuntu = "./images/ubuntu/scripts"
 $PathMacOS = "./images/macos/provision"
 $PatternUbuntu = "#!/bin/bash -e"
 $PatternMacOS = "#!/bin/bash -e -o pipefail"
@@ -34,10 +34,10 @@ if ($ScriptsWithBrokenShebang.Length -gt 0) {
     $ScriptsWithBrokenShebang | ForEach-Object {
         Write-Host "##[error] '$_'"
     }
-    Write-Host "`n`n##[error] Expected shebang for scripts in 'images/linux' folder is '$PatternUbuntu'"
+    Write-Host "`n`n##[error] Expected shebang for scripts in 'images/ubuntu' folder is '$PatternUbuntu'"
     Write-Host "##[error] Expected shebang for scripts in 'images/macos' folder is '$PatternMacOS'"
     exit 1
     else {
         Write-Host "All scripts have correct shebang."
     }
-}
+}

images/ubuntu/scripts/build/Configure-Toolset.ps1

@@ -1,95 +1,95 @@
-################################################################################
+################################################################################
-##  File:  Configure-Toolset.ps1
+##  File:  Configure-Toolset.ps1
-##  Team:  CI-Build
+##  Team:  CI-Build
-##  Desc:  Configure toolset
+##  Desc:  Configure toolset
-################################################################################
+################################################################################
-
+
-Import-Module "$env:HELPER_SCRIPTS/Tests.Helpers.psm1" -DisableNameChecking
+Import-Module "$env:HELPER_SCRIPTS/Tests.Helpers.psm1" -DisableNameChecking
-
+
-function Get-ToolsetToolFullPath
+function Get-ToolsetToolFullPath
-{
+{
-    param
+    param
-    (
+    (
-        [Parameter(Mandatory)] [string] $ToolName,
+        [Parameter(Mandatory)] [string] $ToolName,
-        [Parameter(Mandatory)] [string] $ToolVersion,
+        [Parameter(Mandatory)] [string] $ToolVersion,
-        [Parameter(Mandatory)] [string] $ToolArchitecture
+        [Parameter(Mandatory)] [string] $ToolArchitecture
-    )
+    )
-
+
-    $toolPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath $toolName
+    $toolPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath $toolName
-    $toolPathVersion = Join-Path -Path $toolPath -ChildPath $toolVersion
+    $toolPathVersion = Join-Path -Path $toolPath -ChildPath $toolVersion
-    $foundVersion = Get-Item $toolPathVersion | Sort-Object -Property {[version]$_.name} -Descending | Select-Object -First 1
+    $foundVersion = Get-Item $toolPathVersion | Sort-Object -Property {[version]$_.name} -Descending | Select-Object -First 1
-    $installationDir = Join-Path -Path $foundVersion -ChildPath $toolArchitecture
+    $installationDir = Join-Path -Path $foundVersion -ChildPath $toolArchitecture
-    return $installationDir
+    return $installationDir
-}
+}
-
+
-function Add-EnvironmentVariable
+function Add-EnvironmentVariable
-{
+{
-    param
+    param
-    (
+    (
-        [Parameter(Mandatory)] [string] $Name,
+        [Parameter(Mandatory)] [string] $Name,
-        [Parameter(Mandatory)] [string] $Value,
+        [Parameter(Mandatory)] [string] $Value,
-        [string] $FilePath = "/etc/environment"
+        [string] $FilePath = "/etc/environment"
-    )
+    )
-
+
-    $envVar = "{0}={1}" -f $name, $value
+    $envVar = "{0}={1}" -f $name, $value
-    Tee-Object -InputObject $envVar -FilePath $filePath -Append
+    Tee-Object -InputObject $envVar -FilePath $filePath -Append
-}
+}
-
+
-$ErrorActionPreference = "Stop"
+$ErrorActionPreference = "Stop"
-
+
-Write-Host "Configure toolset tools environment..."
+Write-Host "Configure toolset tools environment..."
-$toolsEnvironment = @{
+$toolsEnvironment = @{
-    go = @{
+    go = @{
-        command = "ln -s {0}/bin/* /usr/bin/"
+        command = "ln -s {0}/bin/* /usr/bin/"
-        variableTemplate = "GOROOT_{0}_{1}_X64"
+        variableTemplate = "GOROOT_{0}_{1}_X64"
-    }
+    }
-}
+}
-
+
-$toolset = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw | ConvertFrom-Json
+$toolset = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw | ConvertFrom-Json
-
+
-foreach ($tool in $toolset.toolcache)
+foreach ($tool in $toolset.toolcache)
-{
+{
-    $toolName = $tool.name
+    $toolName = $tool.name
-    $toolArch = $tool.arch
+    $toolArch = $tool.arch
-    $toolEnvironment = $toolsEnvironment[$toolName]
+    $toolEnvironment = $toolsEnvironment[$toolName]
-
+
-    if (-not $toolEnvironment)
+    if (-not $toolEnvironment)
-    {
+    {
-        continue
+        continue
-    }
+    }
-
+
-    foreach ($toolVersion in $tool.versions)
+    foreach ($toolVersion in $tool.versions)
-    {
+    {
-        Write-Host "Set $toolName $toolVersion environment variable..."
+        Write-Host "Set $toolName $toolVersion environment variable..."
-        $toolPath = Get-ToolsetToolFullPath -ToolName $toolName -ToolVersion $toolVersion -ToolArchitecture $toolArch
+        $toolPath = Get-ToolsetToolFullPath -ToolName $toolName -ToolVersion $toolVersion -ToolArchitecture $toolArch
-        $envName = $toolEnvironment.variableTemplate -f $toolVersion.split(".")
+        $envName = $toolEnvironment.variableTemplate -f $toolVersion.split(".")
-
+
-        # Add environment variable name=value
+        # Add environment variable name=value
-        Add-EnvironmentVariable -Name $envName -Value $toolPath
+        Add-EnvironmentVariable -Name $envName -Value $toolPath
-    }
+    }
-
+
-    # Invoke command and add env variable for the default tool version
+    # Invoke command and add env variable for the default tool version
-    $toolDefVersion = $tool.default
+    $toolDefVersion = $tool.default
-    if (-not $toolDefVersion)
+    if (-not $toolDefVersion)
-    {
+    {
-        continue
+        continue
-    }
+    }
-
+
-    $envDefName = $toolEnvironment.defaultVariable
+    $envDefName = $toolEnvironment.defaultVariable
-    $toolPath = Get-ToolsetToolFullPath -ToolName $toolName -ToolVersion $toolDefVersion -ToolArchitecture $toolArch
+    $toolPath = Get-ToolsetToolFullPath -ToolName $toolName -ToolVersion $toolDefVersion -ToolArchitecture $toolArch
-
+
-    if ($envDefName)
+    if ($envDefName)
-    {
+    {
-        Write-Host "Set default $envDefName for $toolName $toolDefVersion environment variable..."
+        Write-Host "Set default $envDefName for $toolName $toolDefVersion environment variable..."
-        Add-EnvironmentVariable -Name $envDefName -Value $toolPath
+        Add-EnvironmentVariable -Name $envDefName -Value $toolPath
-    }
+    }
-
+
-    if ($toolEnvironment.command)
+    if ($toolEnvironment.command)
-    {
+    {
-        $command = $toolEnvironment.command -f $toolPath
+        $command = $toolEnvironment.command -f $toolPath
-        Write-Host "Invoke $command command for default $toolName $toolDefVersion..."
+        Write-Host "Invoke $command command for default $toolName $toolDefVersion..."
-        Invoke-Expression -Command $command
+        Invoke-Expression -Command $command
-    }
+    }
-}
+}
-
+
-Invoke-PesterTests -TestFile "Toolset" -TestName "Toolset"
+Invoke-PesterTests -TestFile "Toolset" -TestName "Toolset"

images/ubuntu/scripts/build/Install-PowerShellModules.ps1

@@ -1,34 +1,34 @@
-$ErrorActionPreference = "Stop"
+$ErrorActionPreference = "Stop"
-$ProgressPreference = "SilentlyContinue"
+$ProgressPreference = "SilentlyContinue"
-
+
-Import-Module "$env:HELPER_SCRIPTS/Tests.Helpers.psm1" -DisableNameChecking
+Import-Module "$env:HELPER_SCRIPTS/Tests.Helpers.psm1" -DisableNameChecking
-
+
-# Specifies the installation policy
+# Specifies the installation policy
-Set-PSRepository -InstallationPolicy Trusted -Name PSGallery
+Set-PSRepository -InstallationPolicy Trusted -Name PSGallery
-
+
-# Try to update PowerShellGet before the actual installation
+# Try to update PowerShellGet before the actual installation
-Install-Module -Name PowerShellGet -Force
+Install-Module -Name PowerShellGet -Force
-Update-Module -Name PowerShellGet -Force
+Update-Module -Name PowerShellGet -Force
-
+
-# Install PowerShell modules
+# Install PowerShell modules
-$modules = (Get-ToolsetContent).powershellModules
+$modules = (Get-ToolsetContent).powershellModules
-
+
-foreach($module in $modules)
+foreach($module in $modules)
-{
+{
-    $moduleName = $module.name
+    $moduleName = $module.name
-    Write-Host "Installing ${moduleName} module"
+    Write-Host "Installing ${moduleName} module"
-
+
-    if ($module.versions)
+    if ($module.versions)
-    {
+    {
-        foreach ($version in $module.versions)
+        foreach ($version in $module.versions)
-        {
+        {
-            Write-Host " - $version"
+            Write-Host " - $version"
-            Install-Module -Name $moduleName -RequiredVersion $version -Scope AllUsers -SkipPublisherCheck -Force
+            Install-Module -Name $moduleName -RequiredVersion $version -Scope AllUsers -SkipPublisherCheck -Force
-        }
+        }
-        continue
+        continue
-    }
+    }
-
+
-    Install-Module -Name $moduleName -Scope AllUsers -SkipPublisherCheck -Force
+    Install-Module -Name $moduleName -Scope AllUsers -SkipPublisherCheck -Force
-}
+}
-
+
-Invoke-PesterTests -TestFile "PowerShellModules" -TestName "PowerShellModules"
+Invoke-PesterTests -TestFile "PowerShellModules" -TestName "PowerShellModules"

images/ubuntu/scripts/build/preimagedata.sh

@@ -11,7 +11,7 @@ version_major=${os_version/.*/}
 version_wo_dot=${os_version/./}
 github_url="https://github.com/actions/runner-images/blob"
 
-software_url="${github_url}/ubuntu${version_major}/${image_version_major}.${image_version_minor}/images/linux/Ubuntu${version_wo_dot}-Readme.md"
+software_url="${github_url}/ubuntu${version_major}/${image_version_major}.${image_version_minor}/images/ubuntu/Ubuntu${version_wo_dot}-Readme.md"
 releaseUrl="https://github.com/actions/runner-images/releases/tag/ubuntu${version_major}%2F${image_version_major}.${image_version_minor}"
 
 cat <<EOF > $imagedata_file