update_arxiv.pl

update_arxiv.pl

#!/usr/bin/env perl

# expected: s3cmd version 1.6.1 , properly configured
# Run this script from the directory with your existing arXiv download, e.g. /data/arXiv/

use strict;
use warnings;

# Note: you need s3cmd properly setup on your machine first!
my $available = ` s3cmd ls --requester-pays s3://arxiv/src/`;
my @available_tars = grep {defined} map {/(s3\:.+\.tar$)/; $1;} split("\n",$available);

# Obtain already downloaded URLs:
opendir my $cdir, '.';
my %downloaded_tars = map {("s3://arxiv/src/$_" => 1)} grep {/\.tar$/} readdir($cdir);
closedir $cdir;

my @new_tars = grep {!$downloaded_tars{$_};} @available_tars;

foreach my $new_tar(sort @new_tars) {
  print "Fetching: $new_tar\n";
  `s3cmd get --requester-pays $new_tar`; }