scrape_cbc_ca.rb

require 'rubygems'
require 'mechanize'

postal_codes = File.open("postal_codes.txt").read.split("\n")
# randomize to make the pattern slightly harder to see in logs
postal_codes = postal_codes.sort_by {|e| rand(10_000)}

def filename(postcode)
  postcode.sub(' ', '')
end

@agent = WWW::Mechanize.new do |a|
  a.user_agent_alias = 'Mac Safari'
  a.max_history = 1
end

def scrape(postcode)
  fsa, ldu = postcode.split(' ').collect {|e| e.downcase }
  page = @agent.get("http://www.cbc.ca/news/canadavotes/myriding/postalcodes/#{postcode[0..0].downcase}/#{fsa}/#{ldu}.html")
  
  File.open("pages/#{filename(postcode)}", "w") do |f|
    f.puts page.body
  end
end

postal_codes.each do |postcode|
  begin
    next if File.exists?("pages/#{filename(postcode)}")

    scrape(postcode)
    sleep(2 + (rand(3_000) / 1_000.0))
  rescue
    puts "could not get #{postcode}, sleeping for a bit"
    sleep 20
    retry
  end
end