parlscraper

#!/usr/bin/env perl
use strict;
use warnings;
use utf8;
use Web::Scraper;
use URI;
use Hash::Merge qw( merge );
use Data::Dumper;

our $MEMBERS_URL_BASE = 'http://webinfo.parl.gc.ca/MembersOfParliament/';
our $MEMBERS_HTML_URL
    = $MEMBERS_URL_BASE . 'MainMPsCompleteList.aspx?TimePeriod=Current';
our $MEMBER_TABLE_ID
    = 'MasterPage_MasterPage_BodyContent_PageContent_Content_ListContent_ListContent_grdCompleteList';

#MEMBER DETAILS
our $MEMBER_PHOTO_ID
    = 'MasterPage_MasterPage_BodyContent_PageContent_Content_TombstoneContent_TombstoneContent_ucHeaderMP_imgPhoto';
our $MEMBER_TELEPHONE_ID
    = 'MasterPage_MasterPage_BodyContent_PageContent_Content_DetailsContent_DetailsContent__ctl0_lblTelephoneData';
our $MEMBER_EMAIL_ID
    = 'MasterPage_MasterPage_BodyContent_PageContent_Content_DetailsContent_DetailsContent__ctl0_hlEMail';
our $MEMBER_FAX_ID
    = 'MasterPage_MasterPage_BodyContent_PageContent_Content_DetailsContent_DetailsContent__ctl0_lblFaxData';
our $MEMBER_WEBSITE_ID
    = 'MasterPage_MasterPage_BodyContent_PageContent_Content_DetailsContent_DetailsContent__ctl0_hlWebSite';

main();
exit;

sub main {
    my $members = fetch_members();
    warn Dumper $members;
}

sub fetch_members {
    my $members = scrape_members();
    foreach my $member ( @{$members} ) {
        my $member_url = $member->{url};
        next unless $member_url;
        my $member_detail = scrape_member_details($member_url);
        $member = merge($member, $member_detail);
        warn Dumper $member;
    }
    $members;
}

sub scrape_members {
    my $scraper = scraper {
        process '//table[@id="' . $MEMBER_TABLE_ID . '"]/tr',
            'members[]' => scraper {
            process '//td[1]/a', 'name' => 'TEXT';
            process '//td[1]/a', 'url'  => '@href';
            process '//td[1]/a', 'id' => [ '@href', sub {s/.*Key=(.*?)&.*/$1/g} ];
            process '//td[2]/a', 'constituency' => 'TEXT';
            process '//td[3]',   'province'     => 'TEXT';
            process '//td[4]/a', 'caucus'       => 'TEXT';
        };
    };
    my $uri     = new URI($MEMBERS_HTML_URL);
    my $res     = $scraper->scrape($uri);
    my $members = $res->{members};
    $members;
}

sub scrape_member_details {
    my $member_url = shift;

    my $scraper = scraper {
        process '#' . $MEMBER_TELEPHONE_ID, 'telephon' => 'TEXT';
        process '#' . $MEMBER_EMAIL_ID,     'email' => 'TEXT';
        process '#' . $MEMBER_FAX_ID,       'fax'   => 'TEXT';
        process '#' . $MEMBER_WEBSITE_ID,   'website' => [ 'TEXT', sub {s/^/http:\/\//g} ];
        process '#' . $MEMBER_PHOTO_ID,     'profile_photo_url' => '@src';
    };
    my $res = $scraper->scrape($member_url);
    $res;
}

__END__