[Windows] Update functions that get urls and hashes from Github (#8917)

* [Windows] Update function to get hash from github

* Remove redundant function and add documentation

* Fix aws-sam-cli url

images/windows/scripts/build/Install-AWSTools.ps1

@@ -15,11 +15,18 @@ Install-Binary `
 $env:Path = $env:Path + ";$env:ProgramFiles\Amazon\SessionManagerPlugin\bin"
 
 # Install AWS SAM CLI
-$packageName = "AWS_SAM_CLI_64_PY3.msi"
+$downloadUrl = Resolve-GithubReleaseAssetUrl `
-$packageUrl = "https://github.com/awslabs/aws-sam-cli/releases/latest/download/$packageName"
+    -Repo "awslabs/aws-sam-cli" `
-$externalHash = Get-HashFromGitHubReleaseBody -RepoOwner "awslabs" -RepoName "aws-sam-cli" -FileName $packageName
+    -Version "latest" `
+    -UrlMatchPattern "AWS_SAM_CLI_64_PY3.msi"
+$externalHash = Get-GithubReleaseAssetHash `
+    -Repo "awslabs/aws-sam-cli" `
+    -Version "latest" `
+    -FileName (Split-Path $downloadUrl -Leaf) `
+    -HashType "SHA256"
+
 Install-Binary `
-  -Url $packageUrl `
+    -Url $downloadUrl `
     -ExpectedSHA256Sum $externalHash
 
 Invoke-PesterTests -TestFile "CLI.Tools" -TestName "AWS"

images/windows/scripts/build/Install-Git.ps1

@@ -5,12 +5,17 @@
 ################################################################################
 
 # Install the latest version of Git for Windows
-$repoURL = "https://api.github.com/repos/git-for-windows/git/releases/latest"
-$gitReleases = Invoke-RestMethod $repoURL
-$downloadUrl = $gitReleases.assets.browser_download_url -match "Git-.+-64-bit.exe" | Select-Object -First 1
 
-$installerFile = Split-Path $downloadUrl -Leaf
+$downloadUrl = Resolve-GithubReleaseAssetUrl `
-$externalHash = Get-HashFromGitHubReleaseBody -Url $RepoURL -FileName $installerFile
+    -Repo "git-for-windows/git" `
+    -Version "latest" `
+    -UrlMatchPattern "Git-*-64-bit.exe"
+
+$externalHash = Get-GithubReleaseAssetHash `
+    -Repo "git-for-windows/git" `
+    -Version "latest" `
+    -FileName (Split-Path $downloadUrl -Leaf) `
+    -HashType "SHA256"
 
 Install-Binary `
     -Url $downloadUrl `

images/windows/scripts/build/Install-Kotlin.ps1

@@ -9,12 +9,16 @@ $kotlinVersion = (Get-ToolsetContent).kotlin.version
 
 $kotlinDownloadUrl = Resolve-GithubReleaseAssetUrl `
     -Repo "JetBrains/kotlin" `
-    -Version $kotlinVersion `
+    -Version "$kotlinVersion" `
     -Asset "kotlin-compiler-*.zip"
 $kotlinArchivePath = Invoke-DownloadWithRetry $kotlinDownloadUrl
 
 #region Supply chain security
-$externalHash = Get-HashFromGitHubReleaseBody -RepoOwner "JetBrains" -RepoName "kotlin" -FileName "$kotlinBinaryName-*.zip" -Version $kotlinVersion -WordNumber 2
+$externalHash = Get-GithubReleaseAssetHash `
+    -Repo "JetBrains/kotlin" `
+    -Version "$kotlinVersion" `
+    -FileName (Split-Path $kotlinDownloadUrl -Leaf) `
+    -HashType "SHA256"
 Test-FileChecksum $kotlinArchivePath -ExpectedSHA256Sum $externalHash
 #endregion
 

images/windows/scripts/helpers/ImageHelpers.psm1

@@ -22,7 +22,6 @@ Export-ModuleMember -Function @(
     'Install-VSIXFromFile'
     'Install-VSIXFromUrl'
     'Get-VSExtensionVersion'
-    'Get-WinVersion'
     'Test-IsWin22'
     'Test-IsWin19'
     'Install-ChocoPackage'
@@ -38,7 +37,7 @@ Export-ModuleMember -Function @(
     'Get-VisualStudioInstance'
     'Get-VisualStudioComponents'
     'Get-WindowsUpdateStates'
-    'Get-HashFromGitHubReleaseBody'
+    'Get-GithubReleaseAssetHash'
     'Test-FileChecksum'
     'Test-FileSignature'
     'Update-Environment'

images/windows/scripts/helpers/InstallHelpers.ps1

@@ -266,19 +266,66 @@ function Get-TCToolVersionPath {
     return Join-Path $foundVersion $Arch
 }
 
-function Get-WinVersion {
-    (Get-CimInstance -ClassName Win32_OperatingSystem).Caption
-}
-
 function Test-IsWin22 {
-    (Get-WinVersion) -match "2022"
+    <#
+    .SYNOPSIS
+        Checks if the current Windows operating system is Windows Server 2022.
+
+    .DESCRIPTION
+        This function uses the Get-CimInstance cmdlet to retrieve information
+        about the current Windows operating system. It then checks if the Caption
+        property of the Win32_OperatingSystem class contains the string "2022",
+        indicating that the operating system is Windows Server 2022.
+
+    .OUTPUTS
+        Returns $true if the current Windows operating system is Windows Server 2022.
+        Otherwise, returns $false.
+    #>
+    (Get-CimInstance -ClassName Win32_OperatingSystem).Caption -match "2022"
 }
 
 function Test-IsWin19 {
-    (Get-WinVersion) -match "2019"
+    <#
+    .SYNOPSIS
+        Checks if the current Windows operating system is Windows Server 2019.
+
+    .DESCRIPTION
+        This function uses the Get-CimInstance cmdlet to retrieve information
+        about the current Windows operating system. It then checks if the Caption
+        property of the Win32_OperatingSystem class contains the string "2019",
+        indicating that the operating system is Windows Server 2019.
+
+    .OUTPUTS
+        Returns $true if the current Windows operating system is Windows Server 2019.
+        Otherwise, returns $false.
+    #>
+    (Get-CimInstance -ClassName Win32_OperatingSystem).Caption -match "2019"
 }
 
 function Expand-7ZipArchive {
+    <#
+    .SYNOPSIS
+        Extracts files from a 7-Zip archive.
+
+    .DESCRIPTION
+        This function uses the 7z.exe command-line tool to extract files from an archive.
+        The archive path, destination path, and extract method are specified as parameters.
+
+    .PARAMETER Path
+        The path to the archive.
+
+    .PARAMETER DestinationPath
+        The path to the directory where the files will be extracted.
+
+    .PARAMETER ExtractMethod
+        The method used to extract the files.
+        Valid values are "x" (extract with full paths) and "e" (extract without paths).
+
+    .EXAMPLE
+        Expand-7ZipArchive -Path "C:\archive.7z" -DestinationPath "C:\extracted" -ExtractMethod "x"
+
+        Extracts files from the "C:\archive.7z" archive to the "C:\extracted" directory keeping the full paths.
+    #>
     Param
     (
         [Parameter(Mandatory = $true)]
@@ -409,23 +456,60 @@ function Invoke-ScriptBlockWithRetry {
     }
 }
 
-function Resolve-GithubReleaseAssetUrl {
+function Get-GithubReleasesByVersion {
+    <#
+    .SYNOPSIS
+        Retrieves GitHub releases for a specified repository based on version.
+
+    .DESCRIPTION
+        The function retrieves GitHub releases for a specified repository based on the
+        version provided. It supports filtering by version, allowing for the retrieval
+        of specific releases or the latest release. The function utilizes the GitHub REST API
+        to fetch the releases and caches the results to improve performance and reduce
+        the number of API calls.
+
+    .PARAMETER Repository
+        The name of the GitHub repository in the format "owner/repo".
+
+    .PARAMETER Version
+        The version of the release to retrieve. It can be a specific version number,
+        "latest" to retrieve the latest release, or a wildcard pattern to match multiple versions.
+
+    .PARAMETER AllowPrerelease
+        Specifies whether to include prerelease versions in the results. By default,
+        prerelease versions are excluded.
+
+    .EXAMPLE
+        Get-GithubReleasesByVersion -Repository "Microsoft/PowerShell" -Version "7.2.0"
+
+        Retrieves the GitHub releases for the "Microsoft/PowerShell" repository with the version "7.2.0".
+
+    .EXAMPLE
+        Get-GithubReleasesByVersion -Repository "Microsoft/PowerShell" -Version "latest"
+
+        Retrieves the latest GitHub release for the "Microsoft/PowerShell" repository.
+
+    .EXAMPLE
+        Get-GithubReleasesByVersion -Repository "Microsoft/PowerShell" -Version "7.*"
+
+        Retrieves all GitHub releases for the "Microsoft/PowerShell" repository with versions starting with "7.".
+    #>
+
     param (
         [Parameter(Mandatory = $true)]
         [Alias("Repo")]
         [string] $Repository,
-        [Parameter(Mandatory = $true)]
+        [string] $Version = "*",
-        [Alias("Pattern", "File", "Asset")]
+        [switch] $AllowPrerelease
-        [string] $UrlMatchPattern,
-        [switch] $AllowPrerelease,
-        [string] $Version = "*"
     )
 
-    # Add wildcard to the beginning of the pattern if it's not there
+    $localCacheFile = Join-Path ${env:TEMP} "github-releases_$($Repository -replace "/", "_").json"
-    if ($UrlMatchPattern.Substring(0, 2) -ne "*/") {
-        $UrlMatchPattern = "*/$UrlMatchPattern"
-    }
 
+    if (Test-Path $localCacheFile) {
+        $releases = Get-Content $localCacheFile | ConvertFrom-Json
+        Write-Debug "Found cached releases for ${Repository} in local file"
+        Write-Debug "Release count: $($releases.Count)"
+    } else {
         $releases = @()
         $page = 1
         $pageSize = 100
@@ -434,7 +518,11 @@ function Resolve-GithubReleaseAssetUrl {
             $releases += $releasesPage
             $page++
         } while ($releasesPage.Count -eq $pageSize)
+
         Write-Debug "Found $($releases.Count) releases for ${Repository}"
+        Write-Debug "Caching releases for ${Repository} in local file"
+        $releases | ConvertTo-Json -Depth 10 | Set-Content $localCacheFile
+    }
 
     if (-not $releases) {
         throw "Failed to get releases from ${Repository}"
@@ -470,6 +558,63 @@ function Resolve-GithubReleaseAssetUrl {
     }
     Write-Debug "Found $($matchingReleases.Count) releases matching version ${Version} for ${Repository}"
 
+    return $matchingReleases
+}
+
+function Resolve-GithubReleaseAssetUrl {
+    <#
+    .SYNOPSIS
+        Resolves the download URL for a specific asset in a GitHub release.
+
+    .DESCRIPTION
+        This function retrieves the download URL for a specific asset in a GitHub release.
+        It takes the repository name, version, and a URL match pattern as input parameters.
+        It searches for releases that match the specified version and then looks
+        for a download URL that matches the provided pattern. If a matching URL is found,
+        it returns the URL. If no matching URL is found, an exception is thrown.
+
+    .PARAMETER Repository
+        The name of the GitHub repository in the format "owner/repo".
+
+    .PARAMETER Version
+        The version of the release to retrieve. It can be a specific version number,
+        "latest" to retrieve the latest release, or a wildcard pattern to match multiple versions.
+
+    .PARAMETER AllowPrerelease
+        Specifies whether to include prerelease versions in the results. By default,
+        prerelease versions are excluded.
+
+    .PARAMETER UrlMatchPattern
+        The pattern to match against the download URLs of the release assets.
+        Wildcards (*) can be used to match any characters.
+
+    .EXAMPLE
+        Resolve-GithubReleaseAssetUrl -Repository "myrepo" -Version "1.0" -UrlMatchPattern "*.zip"
+        Retrieves the download URL for the asset in the "myrepo" repository with version "1.0" and a file extension of ".zip".
+
+    #>
+
+    param (
+        [Parameter(Mandatory = $true)]
+        [Alias("Repo")]
+        [string] $Repository,
+        [string] $Version = "*",
+        [switch] $AllowPrerelease,
+        [Parameter(Mandatory = $true)]
+        [Alias("Pattern", "File", "Asset")]
+        [string] $UrlMatchPattern
+    )
+
+    $matchingReleases = Get-GithubReleasesByVersion `
+        -Repository $Repository `
+        -AllowPrerelease:$AllowPrerelease `
+        -Version $Version
+
+    # Add wildcard to the beginning of the pattern if it's not there
+    if ($UrlMatchPattern.Substring(0, 2) -ne "*/") {
+        $UrlMatchPattern = "*/$UrlMatchPattern"
+    }
+
     # Loop over releases until we find a download url matching the pattern
     foreach ($release in $matchingReleases) {
         $matchedVersion = $release.version
@@ -493,45 +638,94 @@ function Resolve-GithubReleaseAssetUrl {
     return ($matchedUrl -as [string])
 }
 
-function Get-HashFromGitHubReleaseBody {
+function Get-GithubReleaseAssetHash {
+    <#
+    .SYNOPSIS
+        Retrieves the hash value of a specific file from a GitHub release body.
+
+    .DESCRIPTION
+        The Get-GithubReleaseAssetHash function retrieves the hash value (SHA256 or SHA512)
+        of a specific file from a GitHub release. It searches for the file in the release body
+        and returns the hash value if found.
+
+    .PARAMETER Repository
+        The name of the GitHub repository in the format "owner/repo".
+
+    .PARAMETER Version
+        The version of the release to inspect. It can be a specific version number,
+        "latest" to retrieve the latest release, or a wildcard pattern to match multiple versions.
+
+    .PARAMETER AllowPrerelease
+        Specifies whether to include prerelease versions in the results. By default,
+        prerelease versions are excluded.
+
+    .PARAMETER FileName
+        The name of the file to retrieve the hash value for.
+
+    .PARAMETER HashType
+        The type of hash value to retrieve. Valid values are "SHA256" and "SHA512".
+
+    .EXAMPLE
+        Get-GithubReleaseAssetHash -Repository "MyRepo" -FileName "myfile.txt" -HashType "SHA256"
+
+        Retrieves the SHA256 hash value of "myfile.txt" from the latest release of the "MyRepo" repository.
+
+    .EXAMPLE
+        Get-GithubReleaseAssetHash -Repository "MyRepo" -Version "1.0" -FileName "myfile.txt" -HashType "SHA512"
+
+        Retrieves the SHA512 hash value of "myfile.txt" from the release version "1.0" of the "MyRepo" repository.
+    #>
+
     param (
-        [string] $RepoOwner,
-        [string] $RepoName,
         [Parameter(Mandatory = $true)]
+        [Alias("Repo")]
+        [string] $Repository,
+        [string] $Version = "*",
+        [switch] $AllowPrerelease,
+        [Parameter(Mandatory = $true)]
+        [Alias("File", "Asset")]
         [string] $FileName,
-        [string] $Url,
+        [Parameter(Mandatory = $true)]
-        [string] $Version = "latest",
+        [ValidateSet("SHA256", "SHA512")]
-        [boolean] $IsPrerelease = $false,
+        [string] $HashType
-        [int] $SearchInCount = 100,
-        [string] $Delimiter = '|',
-        [int] $WordNumber = 1
     )
 
-    if ($Url) {
+    $matchingReleases = Get-GithubReleasesByVersion `
-        $releaseUrl = $Url
+        -Repository $Repository `
+        -AllowPrerelease:$AllowPrerelease `
+        -Version $Version
+
+    foreach ($release in $matchingReleases) {
+        $matchedVersion = $release.version
+        $matchedBody = $release.body
+        $matchedLine = $matchedBody.Split("`n") | Where-Object { $_ -like "*$FileName*" }
+        if ($matchedLine.Count -gt 1) {
+            throw "Found multiple lines matching file name '${FileName}' in body of release ${matchedVersion}."
+        } elseif ($matchedLine.Count -ne 0) {
+            break
+        }
+    }
+    if (-not $matchedLine) {
+        throw "File name '${FileName}' not found in release body."
+    }
+    Write-Debug "Found line matching file name '${FileName}' in body of release ${matchedVersion}:`n${matchedLine}"
+
+    if ($HashType -eq "SHA256") {
+        $pattern = "[A-Fa-f0-9]{64}"
+    } elseif ($HashType -eq "SHA512") {
+        $pattern = "[A-Fa-f0-9]{128}"
     } else {
-        if ($Version -eq "latest") {
+        throw "Unknown hash type: ${HashType}"
-            $releaseUrl = "https://api.github.com/repos/${RepoOwner}/${RepoName}/releases/latest"
-        } else {
-            $json = Invoke-RestMethod -Uri "https://api.github.com/repos/${RepoOwner}/${RepoName}/releases?per_page=${SearchInCount}"
-            $tags = $json.Where{ $_.prerelease -eq $IsPrerelease }.tag_name
-            $tag = $tags -match $Version
-            if (-not $tag) {
-                throw "Failed to get a tag name for version $Version."
     }
-            $releaseUrl = "https://api.github.com/repos/${RepoOwner}/${RepoName}/releases/tag/$tag"
+
+    $hash = $matchedLine | Select-String -Pattern $pattern | ForEach-Object { $_.Matches.Value }
+        
+    if ([string]::IsNullOrEmpty($hash)) {
+        throw "Found '${FileName}' in body of release ${matchedVersion}, but failed to get hash from it.`nLine: ${matchedLine}"
     }
-    }
+    Write-Host "Found hash for ${FileName} in release ${matchedVersion}: $hash"
-    $body = (Invoke-RestMethod -Uri $releaseUrl).body -replace ('`', "") -join "`n"
+
-    $matchingLine = $body.Split("`n") | Where-Object { $_ -like "*$FileName*" }    
+    return $hash
-    if ([string]::IsNullOrEmpty($matchingLine)) {
-        throw "File name '$FileName' not found in release body."
-    }
-    $result = $matchingLine.Split($Delimiter)[$WordNumber] -replace "[^a-zA-Z0-9]", ""
-    if ([string]::IsNullOrEmpty($result)) {
-        throw "Empty result. Check Split method parameters (delimiter and/or word number) for the matching line."
-    }
-    return $result
 }
 
 function Test-FileChecksum {