Tools for downloading a lot of files

getstore

#!/usr/bin/perl

use Getopt::Std;
getopts('us') or usage();
use LWP::Simple 'getstore';
use HTTP::Status ();
my $url = shift or usage();
my $file = shift;
unless (defined $file) {
  require File::Basename;
  $file = File::Basename::basename($url);
  $file = "DEFAULT" unless defined $file;
}
my $tmpfile = ".getstore.$$";
END { unlink $tmpfile }

exit if $opt_s && -e $file;

unless (HTTP::Status::is_success(my $rc = getstore($url, $tmpfile))) {
  print HTTP::Status::status_message($rc), "\n";
  exit 1;
}

if (-e $file) {
  if ($opt_u) {
    my ($base, $n, $suffix) = $file =~ /(.*?)(\d*)\.(.*)/;
    $n = 1 unless defined $n;
    do {
      $n++;
      $file = "$base$n.$suffix";
    } while -e $file;
    warn "... Using '$file' instead\n";
  }
}
rename $tmpfile, $file or die "Couldn't rename $tmpfile -> $file: $!";

sub usage {
  print qq{
Usage: $0 [-s] [-u] URL [file]
    -s: if file already exists, exit
    -u: if file already exists, use modified name 
      };
  exit 1;
}

runN

#!/icg/bin/perl

use Getopt::Std;
my %opt = (n => 1);
getopts('r:n:v', \%opt) or usage();

my @cmd = split /\s+/, shift;
@ARGV = shuffle(@ARGV) if $opt{r};

my %pid;
while (@ARGV) {
  if (keys(%pid) < $opt{n}) {
    $pid{spawn(@cmd, shift @ARGV)} = 1;
  } else {
    delete $pid{wait()};
  }
}

1 while wait() >= 0;

sub spawn {
  my $pid = fork;
  die "fork: $!" unless defined $pid;
  return $pid if $pid;
  warn "@_\n" if $opt{v};
  exec @_;
  die "exec: $!";
}

sub usage {
  print STDERR "Usage: $0 [-n N] [-r] [-v] command arg1 arg2...
Run command arg1, command arg2, etc., concurrently.
Run no more than N processes simultaneously (default 1)
    -r: run commands in random order instead of specified order (unimpl.)
    -v: verbose mode
";
  exit 1;
}