Skip to content

Instantly share code, notes, and snippets.

@xaicron
Created December 29, 2009 18:43
Show Gist options
  • Select an option

  • Save xaicron/265482 to your computer and use it in GitHub Desktop.

Select an option

Save xaicron/265482 to your computer and use it in GitHub Desktop.
#!/usr/bin/env perl
use strict;
use warnings;
use Web::Scraper;
use URI;
use WWW::Mechanize;
use File::Basename qw/basename/;
my $user_id = shift || die "Usage: $0 user_id [last_page_num]";
my $last_page = shift;
#===============================================================================
my $uri_base = 'http://drawr.net/';
my $pre_load = scraper {
process '//*[@class="pbooku"]/a/span[2]', num => 'TEXT';
process '//*[@class="pstaru"]/a', id => [ '@href', sub { /[?&]id=(\d+)$/; $1 } ];
};
my $scraper = scraper {
process '//*[@class="entryAreaImg"]', 'list[]' => scraper {
process 'a', id => [ '@href', sub { /[?&]id=(\d+)$|#rid(\d+)$/; $1 || $2 } ];
process 'a>img', url => [ '@src', sub { "$_" } ];
};
};
my $mech = WWW::Mechanize->new(autocheck => 1);
#===============================================================================
&main();exit;
sub main {
mkdir $user_id or die "$user_id $!" unless -d $user_id;
chdir $user_id or die "$user_id $!";
get_images();
}
sub get_images {
my $pre_res = pre_load();
die "Unkown" unless $pre_res->{num} && $pre_res->{id};
my $id = $pre_res->{id};
$last_page ||= int($pre_res->{num} / 10) + ($pre_res->{num} % 10 ? 1 : 0);
for my $page (1..$last_page) {
print "process page $page\n";
my $url = sprintf '%suser.php?id=%d&page=%d', $uri_base, $id, $page;
$mech->get($url);
my $res = $scraper->scrape($mech->content, $mech->uri);
for my $img (@{$res->{list}}) {
save_file($img->{id}, $img->{url});
$mech->add_header(Referer => $url);
}
}
}
sub pre_load {
$mech->get($uri_base . $user_id);
return $pre_load->scrape($mech->content, $mech->uri);
}
sub save_file {
my $id = shift;
my $url = shift;
my $file_name = sprintf '%s_%s', $id, basename $url;
print "skip $file_name\n" and return if -f $file_name;
print "Download $file_name\n";
$mech->get($url);
open my $fh, '>:raw', $file_name or die $!;
print $fh $mech->content;
close $fh;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment