add parsers

azure-pipelines/templates/get-tool-versions-steps.yml

@@ -6,10 +6,7 @@ steps:
     targetType: filePath
     filePath: './get-new-tool-versions/get-new-tool-versions.ps1'
     arguments: |
-        -DistURL "$(DIST_URL)" `
-        -ManifestLink "$(MANIFEST_URL)" `
-        -VersionFilterToInclude $(INCLUDE_FILTER) `
-        -VersionFilterToExclude $(EXCLUDE_FILTER)
+        -ToolName "$(TOOL_NAME)"
 
 - task: PowerShell@2
   displayName: 'Cancel build'

get-new-tool-versions/get-new-tool-versions.ps1

@@ -17,61 +17,24 @@ Optional parameter. Retry count
 #>
 
 param (
-    [Parameter(Mandatory)] [string] $DistURL,
-    [Parameter(Mandatory)] [string] $ManifestLink,
-    [string[]] $VersionFilterToInclude,
-    [string[]] $VersionFilterToExclude,
-    [UInt32] $RetryIntervalSec = 60,
-    [UInt32] $RetryCount = 3
+    [Parameter(Mandatory)] [string] $ToolName
 )
 
-Import-Module (Join-Path $PSScriptRoot "helpers.psm1")
+Import-Module "$PSScriptRoot/parsers/parsers-factory.psm1"
 
-function Get-VersionsByUrl {
-    param (
-        [Parameter(Mandatory)] [string] $ToolPackagesUrl,
-        [Parameter(Mandatory)] [UInt32] $RetryIntervalSec,
-        [Parameter(Mandatory)] [UInt32] $RetryCount
-    )
+$ToolVersionParser = Get-ToolVersionsParser -ToolName $ToolName
+$VersionsFromDist = $ToolVersionParser.GetAvailableVersions()
+$VersionsFromManifest = $ToolVersionParser.GetUploadedVersions()
 
-    $packages = Invoke-RestMethod $ToolPackagesUrl -MaximumRetryCount $RetryCount -RetryIntervalSec $RetryIntervalSec
-    return $packages.version
-}
+Write-Host "Dist"
+$VersionsFromDist | ForEach-Object { Write-Host $_ }
+Write-Host "Manifest"
+$VersionsFromManifest | ForEach-Object { Write-Host $_ }
 
-if ($VersionFilterToInclude) {
-    Validate-FiltersFormat -Filters $VersionFilterToInclude
-}
+$VersionsToBuild = $VersionsFromDist | Where-Object { $VersionsFromManifest -notcontains $_ }
 
-if ($VersionFilterToExclude) {
-    Validate-FiltersFormat -Filters $VersionFilterToExclude
-}
-
-Write-Host "Get the packages list from $DistURL"
-$versionsFromDist = Get-VersionsByUrl -ToolPackagesUrl $DistURL `
-                                      -RetryIntervalSec $RetryIntervalSec `
-                                      -RetryCount $RetryCount
-
-Write-Host "Get the packages list from $ManifestLink"
-[Version[]] $versionsFromManifest = Get-VersionsByUrl -ToolPackagesUrl $ManifestLink `
-                                                      -RetryIntervalSec $RetryIntervalSec `
-                                                      -RetryCount $RetryCount
-
-[Version[]] $formattedVersions = Format-Versions -Versions $versionsFromDist
-
-$formattedVersions = Select-VersionsByFilter -Versions $formattedVersions `
-                                             -IncludeFilters $VersionFilterToInclude `
-                                             -ExcludeFilters $VersionFilterToExclude
-
-if (-not $formattedVersions) {
-    Write-Host "Couldn't find available versions with current filters"
-    exit 1
-}
-
-$versionsToBuild = Skip-ExistingVersions -VersionsFromManifest $versionsFromManifest `
-                                         -VersionsFromDist $formattedVersions
-
-if ($versionsToBuild) {
-    $availableVersions = $versionsToBuild -join ","
+if ($VersionsToBuild) {
+    $availableVersions = $VersionsToBuild -join ","
     $toolVersions = $availableVersions.Replace(",",", ")
     Write-Host "The following versions are available to build:`n$toolVersions"
     Write-Output "##vso[task.setvariable variable=TOOL_VERSIONS;isOutput=true]$toolVersions"

get-new-tool-versions/helpers.psm1

@@ -1,17 +1,3 @@
-function Validate-FiltersFormat {
-    param (
-        [Parameter(Mandatory)] [string[]] $Filters
-    )
-
-    foreach($filter in $Filters) {
-        $filter.Split('.') | ForEach-Object {
-            if (($_ -notmatch '^\d+$') -and ($_ -ne '*')) {
-                throw "Invalid filter format - $filter"
-            }
-        }
-    }
-}
-
 function Format-Versions {
     param (
         [Parameter(Mandatory)] [string[]] $Versions

get-new-tool-versions/parsers/base-parser.psm1

@@ -0,0 +1,32 @@
+class BaseVersionsParser {
+    [Int32]$ApiRetryCount = 3
+    [Int32]$ApiRetryIntervalSeconds = 60
+
+    [SemVer[]] GetAvailableVersions() {
+        $allVersionsRaw = $this.ParseAllAvailableVersions()
+        $allVersions = $allVersionsRaw | ForEach-Object { Write-Host $_; $this.FormatVersion($_) }
+        $filteredVersions = $allVersions | Where-Object { $this.ShouldIncludeVersion($_) }
+        return $filteredVersions
+    }
+
+    [SemVer[]] GetUploadedVersions() {
+        throw "Method is not implemented in base class"
+    }
+
+    hidden [SemVer[]] ParseAllAvailableVersions() {
+        throw "Method is not implemented in base class"
+    }
+
+    hidden [SemVer] FormatVersion([string]$VersionSpec) {
+        throw "Method is not implemented in base class"
+    }
+
+    hidden [bool] ShouldIncludeVersion([SemVer]$Version) {
+        throw "Method is not implemented in base class"
+    }
+
+    hidden [string] BuildGitHubFileUrl($OrganizationName, $RepositoryName, $BranchName, $FilePath) {
+        # https://raw.githubusercontent.com/actions/node-versions/main/versions-manifest.json
+        return "https://raw.githubusercontent.com/${OrganizationName}/${RepositoryName}/${BranchName}/${FilePath}"
+    }
+}

get-new-tool-versions/parsers/go-parser.psm1

@@ -0,0 +1,25 @@
+using module "./base-parser.psm1"
+
+class GoVersionsParser: BaseVersionsParser {
+    [SemVer[]] GetUploadedVersions() {
+        $url = $this.BuildGitHubFileUrl("actions", "go-versions", "main", "versions-manifest.json")
+        $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
+        return $releases.version
+    }
+
+    hidden [string[]] ParseAllAvailableVersions() {
+        $url = "https://golang.org/dl/?mode=json&include=all"
+        $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
+        return $releases.version
+    }
+
+    hidden [SemVer] FormatVersion([string]$VersionSpec) {
+        $cleanVersion = $VersionSpec -replace "^go", ""
+        return [SemVer]$cleanVersion
+    }
+
+    hidden [bool] ShouldIncludeVersion([SemVer]$Version) {
+        # For Go, we include all versions greater than 1.12
+        return $Version -gt [SemVer]"1.12.0"
+    }
+}

get-new-tool-versions/parsers/node-parser.psm1

@@ -0,0 +1,30 @@
+using module "./base-parser.psm1"
+
+class NodeVersionsParser: BaseVersionsParser {
+    [SemVer[]] GetUploadedVersions() {
+        $url = $this.BuildGitHubFileUrl("actions", "node-versions", "main", "versions-manifest.json")
+        $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
+        return $releases.version
+    }
+
+    hidden [string[]] ParseAllAvailableVersions() {
+        $url = "https://nodejs.org/dist/index.json"
+        $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
+        return $releases.version
+    }
+
+    hidden [SemVer] FormatVersion([string]$VersionSpec) {
+        $cleanVersion = $VersionSpec -replace "^v", ""
+        return [SemVer]$cleanVersion
+    }
+
+    hidden [bool] ShouldIncludeVersion([SemVer]$Version) {
+        if ($Version.Major -lt 8) {
+            return $false
+        }
+
+        # For Node.JS, we should include all LTS versions (all even-numbered releases)
+        # https://nodejs.org/en/about/releases/
+        return $Version.Major % 2 -eq 0
+    }
+}

get-new-tool-versions/parsers/parsers-factory.psm1

@@ -0,0 +1,19 @@
+using module "./node-parser.psm1"
+using module "./go-parser.psm1"
+using module "./python-parser.psm1"
+
+function Get-ToolVersionsParser {
+    param(
+        [Parameter(Mandatory)]
+        [string]$ToolName
+    )
+
+    switch ($ToolName) {
+        "Node" { return [NodeVersionsParser]::New() }
+        "Go" { return [GoVersionsParser]::New() }
+        "Python" { return [PythonVersionsParser]::New() }
+        Default {
+            throw "Unknown tool name"
+        }
+    }
+}

get-new-tool-versions/parsers/python-parser.psm1

@@ -0,0 +1,54 @@
+using module "./base-parser.psm1"
+
+class PythonVersionsParser: BaseVersionsParser {
+    [SemVer[]] GetUploadedVersions() {
+        $url = $this.BuildGitHubFileUrl("actions", "python-versions", "main", "versions-manifest.json")
+        $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
+        return $releases.version
+    }
+
+    hidden [string[]] ParseAllAvailableVersions() {
+        $stableVersionsUrl = "https://www.python.org/ftp/python"
+        $stableVersionsHtmlRaw = Invoke-WebRequest $stableVersionsUrl
+        $stableVersionsList = $stableVersionsHtmlRaw.Links.href | Where-Object {
+            $parsed = $null
+            return $_.EndsWith("/") -and [SemVer]::TryParse($_.Replace("/", ""), [ref]$parsed)
+        }
+
+        return $stableVersionsList | ForEach-Object {
+            $subVersionsUrl = "${stableVersionsUrl}/${_}"
+            $subVersionsHtmlRaw = Invoke-WebRequest $subVersionsUrl
+            return $subVersionsHtmlRaw.Links.href | ForEach-Object {
+                if ($_ -match "^Python-(\d+\.\d+\.\d+[a-z]{0,2}\d*)\.tgz$") {
+                    return $Matches[1]
+                }
+            } | ForEach-Object { $_ } | Where-Object { $_ }
+        }
+    }
+
+    hidden [SemVer] FormatVersion([string]$VersionSpec) {
+        $VersionSpec -match "^(\d+)\.(\d+)\.(\d+)([a-z]{1,2})?(\d+)?$"
+        
+        if ($Matches.Count -gt 4) {
+            $VersionLabel = "{0}.{1}" -f $this.ConvertPythonLabel($Matches[4]), $Matches[5]
+            return [SemVer]::new($Matches[1], $Matches[2], $Matches[3], $VersionLabel)
+        }
+
+        return [SemVer]::new($Matches[1], $Matches[2], $Matches[3])
+    }
+
+    hidden [string] ConvertPythonLabel([string]$Label) {
+        switch ($Label) {
+            "a" { return "alpha" }
+            "b" { return "beta" }
+            "rc" { return "rc" }
+        }
+
+        return $Label
+    }
+
+    [bool] ShouldIncludeVersion([SemVer]$Version) {
+        # For Go, we include all versions greater than 1.12
+        return $Version -gt [SemVer]"3.9.0"
+    }
+}