Skip to content

Instantly share code, notes, and snippets.

@mattn
Created March 17, 2009 02:35
Show Gist options
  • Save mattn/80238 to your computer and use it in GitHub Desktop.
Save mattn/80238 to your computer and use it in GitHub Desktop.
use strict;
use warnings;
use Config::Pit;
use WWW::Mechanize;
use Web::Scraper;
use YAML;
my $config = pit_get(
'github.com',
require => {
'username' => 'your username on github.com',
'password' => 'your password on github.com'
}
);
my $mech = WWW::Mechanize->new();
$mech->add_header( 'Accept-Encoding', 'identity' );
$mech->get('https://github.com/login');
$mech->submit_form(
form_number => 2,
fields => {
login => $config->{username},
password => $config->{password},
}
);
my $res = $mech->get('https://github.com/');
my $repos = scraper {
process 'id("repo_listing")/li', 'repos[]' => scraper {
process 'a', title => 'TEXT', url => '@href';
};
result 'repos';
}->scrape($res);
for my $repo ( @{$repos} ) {
my $url = $repo->{url}->as_string;
$url =~ s!/tree$!/network/members!;
$repo->{followers} = scraper {
process 'id("network")//div', 'followers[]' => scraper {
process 'a', title => 'TEXT', url => '@href';
};
result 'followers';
}->scrape( URI->new($url) );
}
warn Dump $repos;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment