download all natgeo's topology maps

download.ps1

$root = "http://pdf.quad.download.s3.amazonaws.com/";
$mroot = $root;
if($pdfs -eq $null -or $pdfs.count -eq 0) {
    $pdfs = @{};
    while(1) {
        $mroot;
        $res = [xml](iwr $mroot | select -ExpandProperty content);

        $res.ListBucketResult.Contents | ?{ [regex]::IsMatch($_.key, "\.pdf`$", "IgnoreCase") } | %{
            try {
                $pdfs.Add($_.key, $_);
            } catch { }
        }
        $marker = $res.ListBucketResult.Contents | select -Last 1
    
        $mroot = $root + "?marker=" + $marker.Key;
    
        if($res.ListBucketResult.Contents.Length -lt 1000) {
            break;
        }
    }
}

mkdir "C:\topologymaps\";

$dsb = {
    param($root, $fn)
    if((Test-Path (Join-Path "C:\topologymaps\" $fn))) {
        return;
    }
    iwr ($root + $fn) -OutFile (Join-Path "C:\topologymaps\" $fn);
};
$djobs = @();
$pdfs.Keys | %{

    Get-Job -State Completed | Remove-Job;
    
    if(($djobs | ?{ $_.State -eq "Running"}).Count -gt 4) {
        ($djobs | ?{ $_.State -eq "Running"}) | Wait-Job -Any
    }
    $tj = Start-Job -ScriptBlock $dsb -ArgumentList @($root, $_);
    $djobs +=($tj);
}