[Ubuntu] Implement new directories hierarchy (#8627)

images/ubuntu/scripts/helpers/Common.Helpers.psm1

@@ -0,0 +1,93 @@
+function Get-CommandResult {
+    param (
+        [Parameter(Mandatory=$true)]
+        [string] $Command,
+        [int[]] $ExpectExitCode = 0,
+        [switch] $Multiline,
+        [bool] $ValidateExitCode = $true
+    )
+
+    # Bash trick to suppress and show error output because some commands write to stderr (for example, "python --version")
+    $stdout = & bash -c "$Command 2>&1"
+    $exitCode = $LASTEXITCODE
+
+    if ($ValidateExitCode) {
+        if ($ExpectExitCode -notcontains $exitCode) {
+            try {
+                throw "StdOut: '$stdout' ExitCode: '$exitCode'"
+            } catch {
+                Write-Host $_.Exception.Message
+                Write-Host $_.ScriptStackTrace
+                exit $LASTEXITCODE
+            }
+        }
+    }
+
+    return @{
+        Output = If ($Multiline -eq $true) { $stdout } else { [string]$stdout }
+        ExitCode = $exitCode
+    }
+}
+
+function Get-OSVersionShort {
+    $(Get-OSVersionFull) | Take-OutputPart -Delimiter '.' -Part 0,1
+}
+
+function Get-OSVersionFull {
+    lsb_release -ds | Take-OutputPart -Part 1, 2
+}
+
+function Get-KernelVersion {
+    $kernelVersion = uname -r
+    return $kernelVersion
+}
+
+function Test-IsUbuntu20 {
+    return (lsb_release -rs) -eq "20.04"
+}
+
+function Test-IsUbuntu22 {
+    return (lsb_release -rs) -eq "22.04"
+}
+
+function Get-ToolsetContent {
+    $toolset = Join-Path $env:INSTALLER_SCRIPT_FOLDER "toolset.json"
+    Get-Content $toolset -Raw | ConvertFrom-Json
+}
+
+function Get-ToolsetValue {
+    param (
+        [Parameter(Mandatory = $true)]
+        [string] $KeyPath
+    )
+
+    $jsonNode = Get-ToolsetContent
+
+    $pathParts = $KeyPath.Split(".")
+    # try to walk through all arguments consequentially to resolve specific json node
+    $pathParts | ForEach-Object {
+        $jsonNode = $jsonNode.$_
+    }
+    return $jsonNode
+}
+
+function Get-AndroidPackages {
+    $packagesListFile = "/usr/local/lib/android/sdk/packages-list.txt"
+
+    if (-Not (Test-Path -Path $packagesListFile -PathType Leaf)) {
+        (/usr/local/lib/android/sdk/cmdline-tools/latest/bin/sdkmanager --list --verbose 2>&1) |
+        Where-Object { $_ -Match "^[^\s]" } |
+        Where-Object { $_ -NotMatch "^(Loading |Info: Parsing |---|\[=+|Installed |Available )" } |
+        Where-Object { $_ -NotMatch "^[^;]*$" } |
+        Out-File -FilePath $packagesListFile
+
+        Write-Host Android packages list:
+        Get-Content $packagesListFile
+    }
+
+    return Get-Content $packagesListFile
+}
+
+function Get-EnvironmentVariable($variable) {
+    return [System.Environment]::GetEnvironmentVariable($variable)
+}

images/ubuntu/scripts/helpers/SoftwareReport.Helpers.psm1

@@ -0,0 +1,41 @@
+function Take-OutputPart {
+    param (
+        [Parameter(ValueFromPipeline)]
+        [string] $toolOutput,
+        [string] $Delimiter = " ",
+        [int[]] $Part
+    )
+    $parts = $toolOutput.Split($Delimiter, [System.StringSplitOptions]::RemoveEmptyEntries)
+    $selectedParts = $parts[$Part]
+    return [string]::Join($Delimiter, $selectedParts)
+}
+
+function Restore-UserOwner {
+    sudo chown -R ${env:USER}: $env:HOME
+}
+
+function Get-LinkTarget {
+    param (
+        [string] $inputPath
+    )
+    $link = Get-Item $inputPath | Select-Object -ExpandProperty Target
+    if ($link) {
+      return " -> $link"
+    }
+    return ""
+}
+
+function Get-PathWithLink {
+    param (
+        [string] $inputPath
+    )
+    $link = Get-LinkTarget($inputPath)
+    return "${inputPath}${link}"
+}
+
+function Get-AptSourceRepository {
+    param([String] $PackageName)
+
+    $sourceUrl = Get-Content "$PSScriptRoot/../helpers/apt-sources.txt" | Select-String -Pattern $PackageName | Take-OutputPart -Part (1..3)
+    return $sourceUrl
+}

images/ubuntu/scripts/helpers/Tests.Helpers.psm1

@@ -0,0 +1,104 @@
+Import-Module "$PSScriptRoot/../helpers/Common.Helpers.psm1" -DisableNameChecking
+
+# Validates that tool is installed and in PATH
+function Validate-ToolExist($tool) {
+    Get-Command $tool -ErrorAction SilentlyContinue | Should -BeTrue
+}
+
+function Invoke-PesterTests {
+    Param(
+        [Parameter(Mandatory)][string] $TestFile,
+        [string] $TestName
+    )
+
+    $testPath = "/imagegeneration/tests/${TestFile}.Tests.ps1"
+    if (-not (Test-Path $testPath)) {
+        throw "Unable to find test file '$TestFile' on '$testPath'."
+    }
+
+    # Check that Pester module is imported
+    if (!(Get-Module "Pester")) {
+        Import-Module Pester
+    }
+
+    $configuration = [PesterConfiguration] @{
+        Run = @{ Path = $testPath; PassThru = $true }
+        Output = @{ Verbosity = "Detailed"; RenderMode = "Plaintext" }
+    }
+    if ($TestName) {
+        $configuration.Filter.FullName = $TestName
+    }
+
+    # Switch ErrorActionPreference to Stop temporary to make sure that tests will fail on silent errors too
+    $backupErrorActionPreference = $ErrorActionPreference
+    $ErrorActionPreference = "Stop"
+    $results = Invoke-Pester -Configuration $configuration
+    $ErrorActionPreference = $backupErrorActionPreference
+
+    # Fail in case if no tests are run
+    if (-not ($results -and ($results.FailedCount -eq 0) -and (($results.PassedCount + $results.SkippedCount) -gt 0))) {
+        $results
+        throw "Test run has failed"
+    }
+}
+
+function ShouldReturnZeroExitCode {
+    Param(
+        [string] $ActualValue,
+        [switch] $Negate,
+        [string] $Because # This parameter is unused but we need it to match Pester asserts signature
+    )
+
+    $result = Get-CommandResult $ActualValue -ValidateExitCode $false
+
+    [bool]$succeeded = $result.ExitCode -eq 0
+    if ($Negate) { $succeeded = -not $succeeded }
+
+    if (-not $succeeded)
+    {
+        $commandOutputIndent = " " * 4
+        $commandOutput = ($result.Output | ForEach-Object { "${commandOutputIndent}${_}" }) -join "`n"
+        $failureMessage = "Command '${ActualValue}' has finished with exit code`n${commandOutput}"
+    }
+
+    return [PSCustomObject] @{
+        Succeeded      = $succeeded
+        FailureMessage = $failureMessage
+    }
+}
+
+function ShouldMatchCommandOutput {
+    Param(
+        [string] $ActualValue,
+        [string] $RegularExpression,
+        [switch] $Negate
+    )
+
+    $output = (Get-CommandResult $ActualValue -ValidateExitCode $false).Output | Out-String
+    [bool] $succeeded = $output -cmatch $RegularExpression
+
+    if ($Negate) {
+        $succeeded = -not $succeeded
+    }
+
+    $failureMessage = ''
+
+    if (-not $succeeded) {
+        if ($Negate) {
+            $failureMessage = "Expected regular expression '$RegularExpression' for '$ActualValue' command to not match '$output', but it did match."
+        }
+        else {
+            $failureMessage = "Expected regular expression '$RegularExpression' for '$ActualValue' command to match '$output', but it did not match."
+        }
+    }
+
+    return [PSCustomObject] @{
+        Succeeded      = $succeeded
+        FailureMessage = $failureMessage
+    }
+}
+
+If (Get-Command -Name Add-ShouldOperator -ErrorAction SilentlyContinue) {
+    Add-ShouldOperator -Name ReturnZeroExitCode -InternalName ShouldReturnZeroExitCode -Test ${function:ShouldReturnZeroExitCode}
+    Add-ShouldOperator -Name MatchCommandOutput -InternalName ShouldMatchCommandOutput -Test ${function:ShouldMatchCommandOutput}
+}

images/ubuntu/scripts/helpers/etc-environment.sh

@@ -0,0 +1,85 @@
+#!/bin/bash -e
+################################################################################
+##  File:  etc-environment.sh
+##  Desc:  Helper functions for source and modify /etc/environment
+################################################################################
+
+# NB: sed expression use '%' as a delimiter in order to simplify handling
+#     values containg slashes (i.e. directory path)
+#     The values containing '%' will break the functions
+
+function getEtcEnvironmentVariable {
+    variable_name="$1"
+    # remove `variable_name=` and possible quotes from the line
+    grep "^${variable_name}=" /etc/environment |sed -E "s%^${variable_name}=\"?([^\"]+)\"?.*$%\1%"
+}
+
+function addEtcEnvironmentVariable {
+    variable_name="$1"
+    variable_value="$2"
+
+    echo "$variable_name=$variable_value" | sudo tee -a /etc/environment
+}
+
+function replaceEtcEnvironmentVariable {
+    variable_name="$1"
+    variable_value="$2"
+
+    # modify /etc/environemnt in place by replacing a string that begins with variable_name
+    sudo sed -i -e "s%^${variable_name}=.*$%${variable_name}=\"${variable_value}\"%" /etc/environment
+}
+
+function setEtcEnvironmentVariable {
+    variable_name="$1"
+    variable_value="$2"
+
+    if grep "$variable_name" /etc/environment > /dev/null; then
+        replaceEtcEnvironmentVariable $variable_name $variable_value
+    else
+        addEtcEnvironmentVariable $variable_name $variable_value
+    fi
+}
+
+function prependEtcEnvironmentVariable {
+    variable_name="$1"
+    element="$2"
+    # TODO: handle the case if the variable does not exist
+    existing_value=$(getEtcEnvironmentVariable "${variable_name}")
+    setEtcEnvironmentVariable "${variable_name}" "${element}:${existing_value}"
+}
+
+function appendEtcEnvironmentVariable {
+    variable_name="$1"
+    element="$2"
+    # TODO: handle the case if the variable does not exist
+    existing_value=$(getEtcEnvironmentVariable "${variable_name}")
+    setEtcEnvironmentVariable "${variable_name}" "${existing_value}:${element}"
+}
+
+function prependEtcEnvironmentPath {
+    element="$1"
+    prependEtcEnvironmentVariable PATH "${element}"
+}
+
+function appendEtcEnvironmentPath {
+    element="$1"
+    appendEtcEnvironmentVariable PATH "${element}"
+}
+
+# Process /etc/environment as if it were shell script with `export VAR=...` expressions
+#    The PATH variable is handled specially in order to do not override the existing PATH
+#    variable. The value of PATH variable read from /etc/environment is added to the end
+#    of value of the exiting PATH variable exactly as it would happen with real PAM app read
+#    /etc/environment
+#
+# TODO: there might be the others variables to be processed in the same way as "PATH" variable
+#       ie MANPATH, INFOPATH, LD_*, etc. In the current implementation the values from /etc/evironments
+#       replace the values of the current environment
+function  reloadEtcEnvironment {
+    # add `export ` to every variable of /etc/environemnt except PATH and eval the result shell script
+    eval $(grep -v '^PATH=' /etc/environment | sed -e 's%^%export %')
+    # handle PATH specially
+    etc_path=$(getEtcEnvironmentVariable PATH)
+    export PATH="$PATH:$etc_path"
+}
+

images/ubuntu/scripts/helpers/install.sh

@@ -0,0 +1,211 @@
+#!/bin/bash -e
+################################################################################
+##  File:  install.sh
+##  Desc:  Helper functions for installing tools
+################################################################################
+
+download_with_retries() {
+# Due to restrictions of bash functions, positional arguments are used here.
+# In case if you using latest argument NAME, you should also set value to all previous parameters.
+# Example: download_with_retries $ANDROID_SDK_URL "." "android_sdk.zip"
+    local URL="$1"
+    local DEST="${2:-.}"
+    local NAME="${3:-${URL##*/}}"
+    local COMPRESSED="$4"
+
+    if [[ $COMPRESSED == "compressed" ]]; then
+        local COMMAND="curl $URL -4 -sL --compressed -o '$DEST/$NAME' -w '%{http_code}'"
+    else
+        local COMMAND="curl $URL -4 -sL -o '$DEST/$NAME' -w '%{http_code}'"
+    fi
+
+    # Save current errexit state and disable it to prevent unexpected exit on error
+    if echo $SHELLOPTS | grep '\(^\|:\)errexit\(:\|$\)' > /dev/null;
+    then
+        local ERR_EXIT_ENABLED=true
+    else
+        local ERR_EXIT_ENABLED=false
+    fi
+    set +e
+
+    echo "Downloading '$URL' to '${DEST}/${NAME}'..."
+    retries=20
+    interval=30
+    while [ $retries -gt 0 ]; do
+        ((retries--))
+        test "$ERR_EXIT_ENABLED" = true && set +e
+        http_code=$(eval $COMMAND)
+        exit_code=$?
+        test "$ERR_EXIT_ENABLED" = true && set -e
+        if [ $http_code -eq 200 ] && [ $exit_code -eq 0 ]; then
+            echo "Download completed"
+            return 0
+        else
+            echo "Error — Either HTTP response code for '$URL' is wrong - '$http_code' or exit code is not 0 - '$exit_code'. Waiting $interval seconds before the next attempt, $retries attempts left"
+            sleep $interval
+        fi
+    done
+
+    echo "Could not download $URL"
+    return 1
+}
+
+## Use dpkg to figure out if a package has already been installed
+## Example use:
+## if ! IsPackageInstalled packageName; then
+##     echo "packageName is not installed!"
+## fi
+IsPackageInstalled() {
+    dpkg -S $1 &> /dev/null
+}
+
+verlte() {
+    sortedVersion=$(echo -e "$1\n$2" | sort -V | head -n1)
+    [  "$1" = "$sortedVersion" ]
+}
+
+get_toolset_path() {
+    echo "/imagegeneration/installers/toolset.json"
+}
+
+get_toolset_value() {
+    local toolset_path=$(get_toolset_path)
+    local query=$1
+    echo "$(jq -r "$query" $toolset_path)"
+}
+
+get_github_package_download_url() {
+    local REPO_ORG=$1
+    local FILTER=$2
+    local VERSION=$3
+    local SEARCH_IN_COUNT="100"
+
+    json=$(curl -fsSL "https://api.github.com/repos/${REPO_ORG}/releases?per_page=${SEARCH_IN_COUNT}")
+
+    if [ -n "$VERSION" ]; then
+        tagName=$(echo $json | jq -r '.[] | select(.prerelease==false).tag_name' | sort --unique --version-sort | egrep -v ".*-[a-z]|beta" | egrep "\w*${VERSION}" | tail -1)
+    else
+        tagName=$(echo $json | jq -r '.[] | select((.prerelease==false) and (.assets | length > 0)).tag_name' | sort --unique --version-sort | egrep -v ".*-[a-z]|beta" | tail -1)
+    fi
+
+    downloadUrl=$(echo $json | jq -r ".[] | select(.tag_name==\"${tagName}\").assets[].browser_download_url | select(${FILTER})" | head -n 1)
+    if [ -z "$downloadUrl" ]; then
+        echo "Failed to parse a download url for the '${tagName}' tag using '${FILTER}' filter"
+        exit 1
+    fi
+    echo $downloadUrl
+}
+
+get_github_package_hash() {
+    local repo_owner=$1
+    local repo_name=$2
+    local file_name=$3
+    local url=$4
+    local version=${5:-"latest"}
+    local prerelease=${6:-false}
+    local delimiter=${7:-'|'}
+    local word_number=${8:-2}
+
+    if [[ -z "$file_name" ]]; then
+        echo "File name is not specified."
+        exit 1
+    fi
+
+    if [[ -n "$url" ]]; then
+        release_url="$url"
+    else
+        if [ "$version" == "latest" ]; then
+            release_url="https://api.github.com/repos/${repo_owner}/${repo_name}/releases/latest"
+        else
+            json=$(curl -fsSL "https://api.github.com/repos/${repo_owner}/${repo_name}/releases?per_page=100")
+            tags=$(echo "$json" | jq -r --arg prerelease "$prerelease" '.[] | select(.prerelease == ($prerelease | test("true"; "i"))) | .tag_name')
+            tag=$(echo "$tags" | grep -o "$version")
+            if [[ "$(echo "$tag" | wc -l)" -gt 1 ]]; then
+                echo "Multiple tags found matching the version $version. Please specify a more specific version."
+                exit 1
+            fi
+            if [[ -z "$tag" ]]; then
+                echo "Failed to get a tag name for version $version."
+                exit 1
+            fi
+            release_url="https://api.github.com/repos/${repo_owner}/${repo_name}/releases/tags/$tag"
+        fi
+    fi
+
+    body=$(curl -fsSL "$release_url" | jq -r '.body' | tr -d '`')
+    matching_line=$(echo "$body" | grep "$file_name")
+    if [[ "$(echo "$matching_line" | wc -l)" -gt 1 ]]; then
+        echo "Multiple lines found included the file $file_name. Please specify a more specific file name."
+        exit 1
+    fi
+    if [[ -z "$matching_line" ]]; then
+        echo "File name '$file_name' not found in release body."
+        exit 1
+    fi
+
+    result=$(echo "$matching_line" | cut -d "$delimiter" -f "$word_number" | tr -d -c '[:alnum:]')
+    if [[ -z "$result" ]]; then
+        echo "Empty result. Check parameters delimiter and/or word_number for the matching line."
+        exit 1
+    fi
+
+    echo "$result"
+}
+
+get_hash_from_remote_file() {
+    local url=$1
+    local keywords=("$2" "$3")
+    local delimiter=${4:-' '}
+    local word_number=${5:-1}
+
+    if [[ -z "${keywords[0]}" || -z "$url" ]]; then
+        echo "File name and/or URL is not specified."
+        exit 1
+    fi
+
+    matching_line=$(curl -fsSL "$url" | sed 's/  */ /g' | tr -d '`')
+    for keyword in "${keywords[@]}"; do
+        matching_line=$(echo "$matching_line" | grep "$keyword")
+    done
+
+    if [[ "$(echo "$matching_line" | wc -l)" -gt 1 ]]; then
+        echo "Multiple lines found including the words: ${keywords[*]}. Please use a more specific filter."
+        exit 1
+    fi
+
+    if [[ -z "$matching_line" ]]; then
+        echo "Keywords (${keywords[*]}) not found in the file with hashes."
+        exit 1
+    fi
+
+    result=$(echo "$matching_line" | cut -d "$delimiter" -f "$word_number" | tr -d -c '[:alnum:]')
+    if [[ ${#result} -ne 64 && ${#result} -ne 128 ]]; then
+        echo "Invalid result length. Expected 64 or 128 characters. Please check delimiter and/or word_number parameters."
+        echo "Result: $result"
+        exit 1
+    fi
+
+    echo "$result"
+}
+
+use_checksum_comparison() {
+    local file_path=$1
+    local checksum=$2
+    local sha_type=${3:-"256"}
+
+    echo "Performing checksum verification"
+
+    if [[ ! -f "$file_path" ]]; then
+        echo "File not found: $file_path"
+        exit 1
+    fi
+
+    local_file_hash=$(shasum --algorithm "$sha_type" "$file_path" | awk '{print $1}')
+
+    if [[ "$local_file_hash" != "$checksum" ]]; then
+        echo "Checksum verification failed. Expected hash: $checksum; Actual hash: $local_file_hash."
+        exit 1
+    else
+        echo "Checksum verification passed"
+    fi
+}

images/ubuntu/scripts/helpers/invoke-tests.sh

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+################################################################################
+##  File:  invoke-tests.sh
+##  Desc:  Helper function for invoking tests
+################################################################################
+
+pwsh -Command "Import-Module '$HELPER_SCRIPTS/Tests.Helpers.psm1' -DisableNameChecking
+    Invoke-PesterTests -TestFile \"$1\" -TestName \"$2\""

images/ubuntu/scripts/helpers/os.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+################################################################################
+##  File:  install-helpers.sh
+##  Desc:  Helper functions for installing tools
+################################################################################
+
+function isUbuntu20
+{
+    lsb_release -d | grep -q 'Ubuntu 20'
+}
+
+function isUbuntu22
+{
+    lsb_release -d | grep -q 'Ubuntu 22'
+}
+
+function getOSVersionLabel
+{
+    lsb_release -cs
+}