Skip to content

Instantly share code, notes, and snippets.

@ispedals
Created May 20, 2013 21:27
Show Gist options
  • Select an option

  • Save ispedals/5615720 to your computer and use it in GitHub Desktop.

Select an option

Save ispedals/5615720 to your computer and use it in GitHub Desktop.
Downloads the files in a google code svn directory by scraping the web interface
#!perl
use v5.12;
use strict;
use warnings;
use WWW::Mechanize;
use Cwd;
my $mech = WWW::Mechanize->new( autocheck => 1 );
my $desktop= "Desktop";
sub urldecode {
use URI::Escape;
my $str= shift;
return uri_unescape($str);
}
sub traverse {
my ($dir, $callback) = @_;
$mech->get( $dir ) if $dir =~ m|(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?| || die "Not valid url: $dir";
my @links=$mech->links();
shift @links;
for (grep { $_->url() !~ /^http/ } @links) {
my $path=$_->url();
say "Current path: $path";
&$callback($_);
traverse($_->base().$path, $callback) if $path =~ /\/$/;
chdir '..' if $path =~ /\/$/;
say 'returned to '. cwd();
}
return();
}
sub make_folder {
my $file=shift;
my $files=urldecode($file->url());
if ($files =~ /\/$/) { #if directory
chop $files;
my $directory=cwd().'/'.$files;
mkdir $directory,755;
chdir $directory;
say "Created directory: $files";
return;
}
my $link=$file->base().$file->url();
say "Created file: $link in ".cwd();
$mech->get( $link, ":content_file" => &urldecode($file->url()) );
}
my $url= $ARGV[0];
$url =~ m|.*/(.*?)/$|;
my $folde r= urldecode($1);
chdir $desktop;
say "Making directory: $desktop/$folder";
mkdir "$desktop/$folder",755;
chdir "$desktop/$folder";
say "current directory ".cwd();
traverse($url, \&make_folder);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment