Skip to content

Instantly share code, notes, and snippets.

@briandfoy
Created September 24, 2018 21:32
Show Gist options
  • Save briandfoy/ddef83c6330133dc3ad5996b4be961f8 to your computer and use it in GitHub Desktop.
Save briandfoy/ddef83c6330133dc3ad5996b4be961f8 to your computer and use it in GitHub Desktop.
(Perl) (mojolicious) scrape an imgur gallery
#!/Users/brian/bin/perls/perl-latest
use v5.26;
use utf8;
use strict;
use warnings;
use Mojo::File qw(path);
use Mojo::UserAgent;
use Mojo::Util qw(dumper);
my $ua = Mojo::UserAgent->new;
my $client_id = $ENV{IMGUR_CLIENT_ID}
|| die "Set IMGUR_CLIENT_ID to use this program\n";
my $extra_request_headers = {
Authorization => "Client-ID $client_id",
};
@ARGV = 'dogs' unless @ARGV;
foreach my $subreddit ( @ARGV ) {
say "============ $subreddit";
my $url = "https://api.imgur.com/3/gallery/r/$subreddit";
my $dir = path( $subreddit )->make_path;
my $tx = $ua->get(
$url => $extra_request_headers
);
if( $tx->error ) {
warn "Could not fetch $subreddit. Skipping...\n";
next;
}
foreach my $hash ( $tx->result->json->{data}->@* ) {
my @hashes =
exists $hash->{images_count} ? $hash->{images}->@* : $hash;
my @links = grep { defined } map { $_->@{qw(link mp4)} } @hashes;
say "Links: @links";
foreach my $link ( @links ) {
my $basename = Mojo::URL->new( $link )->path->parts->[0];
my $path = $dir->child( $basename );
if( -e $path ) {
say "Already have $path: skipping";
next;
}
my $tx = $ua->get( $link => $extra_request_headers );
if( $tx->error ) {
warn "Error fetching " . $tx->req->url . "\n";
next;
}
$tx->res->content->asset->move_to( $path );
utime $hash->{datetime}, $hash->{datetime}, $path;
sleep 1;
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment