Search for packages not on the PowerShell Gallery

Find-MissingPSGalleryPackages.ps1

using namespace System.Management.Automation
using namespace System.Collections.Generic
using namespace System.Collections.Concurrent

$baseUri = 'https://www.powershellgallery.com/api/v2/Packages?$orderby=Published desc&$skip='
$maxPackages = 700000
$interval = 100
$statusDictionary = [ConcurrentDictionary[string, bool]]::new()

$existingpackagesjson = (iwr 'https://pwsh.gallery/sleet.packageindex.json').content -replace 'PSObject','__MODULEFASTIGNOREME' | convertfrom-json |% packages
# Convert the psobject existingpackages into a hashtable for faster lookups
$existingPackages = @{}
foreach ($package in $existingpackagesjson.psobject.properties) {
	try {
		$existingPackages[$package.Name] = $package.Value
	} catch {
		#These are just builtin properties we dont care about
	}
}

$packages = 1..($maxPackages / $interval) | ForEach-Object -ThrottleLimit 1 -Parallel {
	$status = $using:statusdictionary

	if ($status['endreached']) {
		Write-Host -ForegroundColor Green "End reached. Stopping further processing."
		return
	}

	Write-Progress -Activity "Fetching Packages" -Status "Processing $($_ * $using:interval) of $($using:maxPackages)" -PercentComplete (($_ * $using:interval) / $using:maxPackages * 100)

	$skip = $_ * $using:interval

	$processing = $true
	while ($processing) {
		try {
			$irmParams = @{
				Uri  = 'https://www.powershellgallery.com/api/v2/Packages'
				Body = @{
					'$orderby'        = 'Published desc'
					'$skip'           = $skip
					'$top'            = 100
				}
			}
			$result = Invoke-RestMethod @irmParams -OperationTimeoutSeconds 300 -ConnectionTimeoutSeconds 300
		} catch {
			$transientError = $false
			$statusCode = $null

			if ($_.Exception.PSObject.Properties['Response'] -and $_.Exception.Response) {
				try {
					$statusCode = [int]$_.Exception.Response.StatusCode
				} catch {
					$statusCode = $null
				}
			}

			if ($null -eq $statusCode) {
				# no response / transient network condition
				$transientError = $true
			} elseif ($statusCode -eq 429 -or ($statusCode -ge 500 -and $statusCode -le 599)) {
				# throttling and server-side failures
				$transientError = $true
			}

			if ($transientError) {
				Write-Host "Packages $skip`: Transient error occurred ($($statusCode ?? 'Timeout')). Retrying..."
				Start-Sleep -Milliseconds 500
				continue
			}

			throw "Unrecoverable error fetching packages at skip $skip`: $($_.Exception.Message)"
		}
		$processing = $false
	}
	
	if ($result.Count -eq 0) {
		Write-Host -ForegroundColor Green "No more packages found at skip $skip. Ending processing."
		$status['endreached'] = $true
	}
	return $result | Foreach-Object {
		[PSCustomObject]@{
			Id = $_.Properties.Id
			Version = $_.Properties.Version
			NormalizedVersion = $_.Properties.NormalizedVersion
		}
	}
} | Foreach-Object {
	if ($null -eq $existingPackages[$_.Id]) {
		Write-Host -ForegroundColor Yellow "Package $($_.Id) $($_.NormalizedVersion) not found in existing packages."
		return $_
	}

	$versions = $existingPackages[$_.Id]
	if ($_.NormalizedVersion -notin $versions) {
		Write-Host -ForegroundColor Yellow "Package $($_.Id) $($_.NormalizedVersion) has a new or missing version."
		return $_
	}
} | Tee-Object -Variable newPackages