Merge helpers function from node, boost, python. (#2)

* setup boost helpers * add ci for pull requests * pester for ci * resolving issue with pester 5 * remove Output flag * check tests with 4 version * try common tests variant * added requires * move to 5 version * try to add assert module * fix scope * use new regex for python * fix regex in tests * Pester 4.10.1 * EnableExit * fix synopsys * fix creating tar archive Co-authored-by: Dmitry Shibanov <v-dmshib@microsoft.com> Co-authored-by: Maxim Lobanov <v-malob@microsoft.com>
2025-12-10 03:13:23 +00:00 · 2020-06-01 17:46:35 +03:00
parent 8affb5df76
commit d8c3ce72ee
13 changed files with 422 additions and 184 deletions
--- a/packages-generation/common-helpers.psm1
+++ b/packages-generation/common-helpers.psm1
@@ -1,80 +0,0 @@
-<#
-.SYNOPSIS
-The execute command and print all output to the logs
-#>
-function Execute-Command {
-    [CmdletBinding()]
-    param(
-        [Parameter(Mandatory=$true)][string] $Command
-    )
-
-    Write-Debug "Execute $Command"
-
-    try {
-        Invoke-Expression $Command | ForEach-Object { Write-Host $_ }
-    }
-    catch {
-        Write-Host "Error happened during command execution: $Command"
-        Write-Host "##vso[task.logissue type=error;] $_"
-    }
-}
-
-<#
-.SYNOPSIS
-Download file from url and return local path to file
-#>
-function Download-File {
-    param(
-        [Parameter(Mandatory=$true)]
-        [Uri]$Uri,
-        [Parameter(Mandatory=$true)]
-        [String]$OutputFolder
-    )
-
-    $targetFilename = [IO.Path]::GetFileName($Uri)
-    $targetFilepath = Join-Path $OutputFolder $targetFilename
-
-    Write-Debug "Download source from $Uri to $OutFile"
-    try {
-        (New-Object System.Net.WebClient).DownloadFile($Uri, $targetFilepath)
-        return $targetFilepath
-    } catch {
-        Write-Host "Error during downloading file from '$Uri'"
-        "$_"
-        exit 1
-    }    
-}
-
-<#
-.SYNOPSIS
-Generate file that contains the list of all files in particular directory
-#>
-function New-ToolStructureDump {
-    param(
-        [Parameter(Mandatory=$true)]
-        [String]$ToolPath,
-        [Parameter(Mandatory=$true)]
-        [String]$OutputFolder
-    )
-
-    $outputFile = Join-Path $OutputFolder "tools_structure.txt"
-
-    $folderContent = Get-ChildItem -Path $ToolPath -Recurse | Sort-Object | Select-Object -Property FullName, Length
-    $folderContent | ForEach-Object {
-        $relativePath = $_.FullName.Replace($ToolPath, "");
-        return "${relativePath}"
-    } | Out-File -FilePath $outputFile
-}
-
-<#
-.SYNOPSIS
-Check if it is macOS / Ubuntu platform
-#>
-function IsNixPlatform {
-    param(
-        [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()]
-        [String]$Platform
-    )
-
-    return ($Platform -match "macos") -or ($Platform -match "ubuntu")
-}
--- a/packages-generation/generate-versions-manifest.ps1
+++ b/packages-generation/generate-versions-manifest.ps1
@@ -1,145 +0,0 @@
-<#
-.SYNOPSIS
-Generate versions manifest based on repository releases
-
-.DESCRIPTION
-Versions manifest is needed to find the latest assets for particular version of tool
-.PARAMETER GitHubRepositoryOwner
-Required parameter. The organization which tool repository belongs
-.PARAMETER GitHubRepositoryName
-Optional parameter. The name of tool repository
-.PARAMETER GitHubAccessToken
-Required parameter. PAT Token to overcome GitHub API Rate limit
-.PARAMETER OutputFile
-Required parameter. File "*.json" where generated results will be saved
-.PARAMETER PlatformMapFile
-Optional parameter. Path to the json file with platform map
-Structure example:
-{
-    "macos-1014": [
-        {
-            "platform": "darwin",
-            "platform_version": "10.14"
-        }, ...
-    ], ...
-}
-#>
-
-param (
-    [Parameter(Mandatory)] [string] $GitHubRepositoryOwner,
-    [Parameter(Mandatory)] [string] $GitHubRepositoryName,
-    [Parameter(Mandatory)] [string] $GitHubAccessToken,
-    [Parameter(Mandatory)] [string] $OutputFile,
-    [string] $PlatformMapFile
-)
-
-Import-Module (Join-Path $PSScriptRoot "../github/github-api.psm1")
-
-if ($PlatformMapFile -and (Test-Path $PlatformMapFile)) {
-    $PlatformMap = Get-Content $PlatformMapFile -Raw | ConvertFrom-Json -AsHashtable
-} else {
-    $PlatformMap = @{}
-}
-
-function New-AssetItem {
-    param (
-        [Parameter(Mandatory)][string]$Filename,
-        [Parameter(Mandatory)][string]$DownloadUrl,
-        [Parameter(Mandatory)][string]$Arch,
-        [Parameter(Mandatory)][string]$Platform,
-        [string]$PlatformVersion
-    )
-    $asset = New-Object PSObject
-
-    $asset | Add-Member -Name "filename" -Value $Filename -MemberType NoteProperty
-    $asset | Add-Member -Name "arch" -Value $Arch -MemberType NoteProperty
-    $asset | Add-Member -Name "platform" -Value $Platform -MemberType NoteProperty
-    if ($PlatformVersion) { $asset | Add-Member -Name "platform_version" -Value $PlatformVersion -MemberType NoteProperty }
-    $asset | Add-Member -Name "download_url" -Value $DownloadUrl -MemberType NoteProperty
-
-    return $asset
-}
-
-function Build-AssetsList {
-    param (
-        [AllowEmptyCollection()]
-        [Parameter(Mandatory)][array]$ReleaseAssets
-    )
-
-    
-    $assets = @()
-    foreach($releaseAsset in $ReleaseAssets) {
-        $parts = [IO.path]::GetFileNameWithoutExtension($releaseAsset.name).Split("-")
-        $arch = $parts[-1]
-        $buildPlatform = [string]::Join("-", $parts[2..($parts.Length-2)])
-
-        if ($PlatformMap[$buildPlatform]) {
-            $PlatformMap[$buildPlatform] | ForEach-Object {
-                $assets += New-AssetItem -Filename $releaseAsset.name `
-                                         -DownloadUrl $releaseAsset.browser_download_url `
-                                         -Arch $arch `
-                                         -Platform $_.platform `
-                                         -PlatformVersion $_.platform_version
-            }
-
-        } else {
-            $assets += New-AssetItem -Filename $releaseAsset.name `
-                                     -DownloadUrl $releaseAsset.browser_download_url `
-                                     -Arch $arch `
-                                     -Platform $buildPlatform
-        }
-    }
-
-    return $assets
-}
-
-function Get-VersionFromRelease {
-    param (
-        [Parameter(Mandatory)][object]$Release
-    )
-    # Release name can contain additional information after ':' so filter it
-    [string]$releaseName = $Release.name.Split(':')[0]
-    [Version]$version = $null
-    if (![Version]::TryParse($releaseName, [ref]$version)) {
-        throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )"
-    }
-
-    return $version
-}
-
-function Build-VersionsManifest {
-    param (
-        [Parameter(Mandatory)][array]$Releases
-    )
-
-    $Releases = $Releases | Sort-Object -Property "published_at" -Descending
-
-    $versionsHash = @{}
-    foreach ($release in $Releases) {
-        if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) {
-            continue
-        }
-
-        [Version]$version = Get-VersionFromRelease $release
-        $versionKey = $version.ToString()
-
-        if ($versionsHash.ContainsKey($versionKey)) {
-            continue
-        }
-
-        $versionsHash.Add($versionKey, [PSCustomObject]@{
-            version = $versionKey
-            stable = $true
-            release_url = $release.html_url
-            files = Build-AssetsList $release.assets
-        })
-    }
-
-    # Sort versions by descending
-    return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true }
-}
-
-$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken
-$releases = $gitHubApi.GetGitHubReleases()
-$versionIndex = Build-VersionsManifest $releases
-$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding utf8 -Force
--- a/packages-generation/manifest-generator.ps1
+++ b/packages-generation/manifest-generator.ps1
@@ -0,0 +1,35 @@
+<#
+.SYNOPSIS
+Generate versions manifest based on repository releases
+
+.DESCRIPTION
+Versions manifest is needed to find the latest assets for particular version of tool
+.PARAMETER GitHubRepositoryOwner
+Required parameter. The organization which tool repository belongs
+.PARAMETER GitHubRepositoryName
+Required parameter. The name of tool repository
+.PARAMETER GitHubAccessToken
+Required parameter. PAT Token to overcome GitHub API Rate limit
+.PARAMETER OutputFile
+Required parameter. File "*.json" where generated results will be saved
+.PARAMETER ConfigurationFile
+Path to the json file with parsing configuration
+#>
+
+param (
+    [Parameter(Mandatory)] [string] $GitHubRepositoryOwner,
+    [Parameter(Mandatory)] [string] $GitHubRepositoryName,
+    [Parameter(Mandatory)] [string] $GitHubAccessToken,
+    [Parameter(Mandatory)] [string] $OutputFile,
+    [Parameter(Mandatory)] [string] $ConfigurationFile
+)
+
+Import-Module (Join-Path $PSScriptRoot "../github/github-api.psm1")
+Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force
+
+$configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
+
+$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken
+$releases = $gitHubApi.GetReleases()
+$versionIndex = Build-VersionsManifest -Releases $releases -Configuration $configuration
+$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force
--- a/packages-generation/manifest-utils.Tests.ps1
+++ b/packages-generation/manifest-utils.Tests.ps1
@@ -0,0 +1,119 @@
+#Requires -Modules Pester
+#Requires -Modules Assert
+
+Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force
+  
+Describe "New-AssetItem" {
+    It "use regex to parse all values in correct order" {
+        $githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; }
+        $configuration = @{
+            regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
+            groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; };
+        }
+        $expectedOutput = [PSCustomObject]@{
+            filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04";
+            arch = "x64"; download_url = "long_url";
+        }
+
+        $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration
+        Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput
+    }
+
+    It "support constant values in groups" {
+        $githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; }
+        $configuration = @{
+            regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
+            groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; }
+        }
+        $expectedOutput = [PSCustomObject]@{
+            filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04";
+            arch = "x64"; download_url = "long_url";
+        }
+
+        $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration
+        Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput
+    }
+
+    It "Skip empty groups" {
+        $githubAsset = @{ name = "python-3.8.3-win32-x64.zip"; browser_download_url = "long_url"; }
+        $configuration = @{
+            regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
+            groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; }
+        }
+        $expectedOutput = [PSCustomObject]@{
+            filename = "python-3.8.3-win32-x64.zip"; platform = "win32";
+            arch = "x64"; download_url = "long_url";
+        }
+
+        $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration
+        Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput
+    }
+}
+
+Describe "Get-VersionFromRelease" {
+    It "clear version" {
+        $release = @{ name = "3.8.3" }
+        Get-VersionFromRelease -Release $release | Should -Be "3.8.3"
+    }
+
+    It "version with title" {
+        $release = @{ name = "3.8.3: Release title" }
+        Get-VersionFromRelease -Release $release | Should -Be "3.8.3"
+    }
+}
+
+Describe "Build-VersionsManifest" {
+    $assets = @(
+        @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "fake_url"; }
+        @{ name = "python-3.8.3-linux-18.04-x64.tar.gz"; browser_download_url = "fake_url"; }
+    )
+    $configuration = @{
+        regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
+        groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; }
+    }
+    $expectedManifestFiles = @(
+        [PSCustomObject]@{ filename = "python-3.8.3-linux-16.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "16.04"; download_url = "fake_url" },
+        [PSCustomObject]@{ filename = "python-3.8.3-linux-18.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "18.04"; download_url = "fake_url" }
+    )
+
+    It "build manifest with correct version order" {
+        $releases = @(
+            @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets },
+            @{ name = "3.5.2: Hello"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets },
+            @{ name = "3.8.3: Release title"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets }
+        )
+        $expectedManifest = @(
+            [PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles },
+            [PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles },
+            [PSCustomObject]@{ version = "3.5.2"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }
+        )
+        $actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
+        Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest
+    }
+
+    It "Skip draft and prerelease" {
+        $releases = @(
+            @{ name = "3.8.1"; draft = $true; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets },
+            @{ name = "3.5.2"; draft = $false; prerelease = $true; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets },
+            @{ name = "3.8.3"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets }
+        )
+        $expectedManifest = @(
+            [PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }
+        )
+        [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
+        Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest 
+    }
+
+    It "take latest published release for each version" {
+        $releases = @(
+            @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url1"; published_at = "2020-05-06T11:45:36Z"; assets = $assets },
+            @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url2"; published_at = "2020-05-14T09:54:06Z"; assets = $assets },
+            @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url3"; published_at = "2020-05-06T11:43:38Z"; assets = $assets }
+        )
+        $expectedManifest = @(
+            [PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url2"; files = $expectedManifestFiles }
+        )
+        [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
+        Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest
+    }
+}
--- a/packages-generation/manifest-utils.psm1
+++ b/packages-generation/manifest-utils.psm1
@@ -0,0 +1,77 @@
+function Read-ConfigurationFile {
+    param ([Parameter(Mandatory)][string]$Filepath)
+    return Get-Content $Filepath -Raw | ConvertFrom-Json
+}
+
+function New-AssetItem {
+    param (
+        [Parameter(Mandatory)][object]$ReleaseAsset,
+        [Parameter(Mandatory)][object]$Configuration
+    )
+    $regexResult = [regex]::Match($ReleaseAsset.name, $Configuration.regex)
+    if (-not $regexResult.Success) { throw "Can't match asset filename '$($_.name)' to regex" }
+
+    $result = New-Object PSObject
+    $result | Add-Member -Name "filename" -Value $ReleaseAsset.name -MemberType NoteProperty
+    $Configuration.groups.PSObject.Properties | ForEach-Object {
+        if (($_.Value).GetType().Name.StartsWith("Int")) {
+            $value = $regexResult.Groups[$_.Value].Value
+        } else {
+            $value = $_.Value
+        }
+
+        if (-not ([string]::IsNullOrEmpty($value))) {
+            $result | Add-Member -Name $_.Name -Value $value -MemberType NoteProperty
+        }
+    }
+
+    $result | Add-Member -Name "download_url" -Value $ReleaseAsset.browser_download_url -MemberType NoteProperty
+    return $result
+}
+
+function Get-VersionFromRelease {
+    param (
+        [Parameter(Mandatory)][object]$Release
+    )
+    # Release name can contain additional information after ':' so filter it
+    [string]$releaseName = $Release.name.Split(':')[0]
+    [Version]$version = $null
+    if (![Version]::TryParse($releaseName, [ref]$version)) {
+        throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )"
+    }
+
+    return $version
+}
+
+function Build-VersionsManifest {
+    param (
+        [Parameter(Mandatory)][array]$Releases,
+        [Parameter(Mandatory)][object]$Configuration
+    )
+
+    $Releases = $Releases | Sort-Object -Property "published_at" -Descending
+
+    $versionsHash = @{}
+    foreach ($release in $Releases) {
+        if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) {
+            continue
+        }
+
+        [Version]$version = Get-VersionFromRelease $release
+        $versionKey = $version.ToString()
+
+        if ($versionsHash.ContainsKey($versionKey)) {
+            continue
+        }
+
+        $versionsHash.Add($versionKey, [PSCustomObject]@{
+            version = $versionKey
+            stable = $true
+            release_url = $release.html_url
+            files = $release.assets | ForEach-Object { New-AssetItem -ReleaseAsset $_ -Configuration $Configuration }
+        })
+    }
+
+    # Sort versions by descending
+    return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true }
+}
--- a/packages-generation/nix-helpers.psm1
+++ b/packages-generation/nix-helpers.psm1
@@ -1,34 +0,0 @@
-<#
-.SYNOPSIS
-Pack folder to *.zip format
-#>
-function Pack-Zip {
-    param(
-        [Parameter(Mandatory=$true)]
-        [String]$PathToArchive,
-        [Parameter(Mandatory=$true)]
-        [String]$ToolZipFile
-    )
-
-    Write-Debug "Pack $PathToArchive to $ToolZipFile"
-    Push-Location -Path $PathToArchive
-    zip -q -r $ToolZipFile * | Out-Null
-    Pop-Location
-}
-
-<#
-.SYNOPSIS
-Unpack *.tar file
-#>
-function Unpack-TarArchive {
-    param(
-        [Parameter(Mandatory=$true)]
-        [String]$ArchivePath,
-        [Parameter(Mandatory=$true)]
-        [String]$OutputDirectory
-    )
-
-    Write-Debug "Unpack $ArchivePath to $OutputDirectory"
-    tar -C $OutputDirectory -xzf $ArchivePath
-
-}
--- a/packages-generation/pester-extensions.psm1
+++ b/packages-generation/pester-extensions.psm1
@@ -1,33 +0,0 @@
-<#
-.SYNOPSIS
-Pester extension that allows to run command and validate exit code
-.EXAMPLE
-"python file.py" | Should -ReturnZeroExitCode
-#>
-function ShouldReturnZeroExitCode {
-    Param(
-        [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
-        [String]$ActualValue,
-        [switch]$Negate
-    )
-
-    Write-Host "Run command '${ActualValue}'"
-    Invoke-Expression -Command $ActualValue | ForEach-Object { Write-Host $_ }
-    $actualExitCode = $LASTEXITCODE
-
-    [bool]$succeeded = $actualExitCode -eq 0
-    if ($Negate) { $succeeded = -not $succeeded }
-
-    if (-not $succeeded)
-    {
-        $failureMessage = "Command '${ActualValue}' has finished with exit code ${actualExitCode}"
-    }
-
-    return New-Object PSObject -Property @{
-        Succeeded      = $succeeded
-        FailureMessage = $failureMessage
-    }
-}
-
-Add-AssertionOperator -Name ReturnZeroExitCode `
-                    -Test  $function:ShouldReturnZeroExitCode