A script to scrape what nytimes.com recommends to me and also, what is popular among other users

nyt-list-scrape.rb

#!/usr/bin/env ruby
# nyt-list-scrape.rb
# Dan Nguyen @dancow
# Created: 2014-07-26
# Updated: 2014-07-29
#
# Scrape the titles/urls of articles recommended to the user by the NYT,
# and also, the most popular ones at the moment
#
# Requirements: Ruby 1.9.x and the Mechanize gem and a NYT digital subscriber account
#
# Usage:
# Run the script at the commandline and pass in two arguments: user_id and password
#
#     ruby nyt-list-scrape.rb [email protected] mypassword123 (third arg can be 1 to turn verbose mode)
#
# This will create a subdirectory called "./nyt-list-scrapes"
# with a timestamped CSV:  "./nyt-list-scrapes/2014-07-26_10_30.csv"
#
# in this format:
#
# headline,rank,list,period,url,timestamp_collected
# "C.D.C. Says Hello",1,recommendations,,http://www.nytimes.com/2014/07/25/health/cdc-says-hello,2014-07-22 10:30
# ...
# "C.D.C. Says Hello",4,most-popular-twitter,7,http://www.nytimes.com/2014/07/25/health/cdc-says-hello,2014-07-22 10:30
#
require 'mechanize'
require 'open-uri'
require 'csv'

my_user_id = ARGV[0]
my_password = ARGV[1]
is_verbose = ARGV[2].to_i == 1

## Initialize agent
agent = Mechanize.new { |a| a.user_agent_alias = 'Mac Safari' }
agent.get('https://myaccount.nytimes.com/auth/login') do |login|
  login.form.userid = my_user_id
  login.form.password = my_password
  login.form.submit
end

puts "Successfully authenticated as #{my_user_id}" if is_verbose

timestamp = Time.now

# open up CSV
subdir = File.expand_path '../nyt-list-scrapes', __FILE__
Dir.mkdir(subdir) unless Dir.exists?(subdir)
fname = File.join subdir, "#{timestamp.strftime('%Y-%m-%d_%H_%M')}.csv"
csv = CSV.open(fname, 'w', headers: true)
csv << %w(headline rank list period url timestamp_collected)

puts "Saving to #{fname}" if is_verbose

# get recommendations and recently-read, as
# they both exist on the same page
list_url = "http://www.nytimes.com/recommendations"
puts '', "-----------", list_url  if is_verbose if is_verbose
agent.get(list_url) do |list_page|
  ## first get recommendations
  list_page.search('.story h3 a').each_with_index do |a, rank|
    title = a.text.gsub(/\s+/, ' ').strip
    base_href = a.attr('href').sub(/\?.+$/, '') # don't need the analytics parameters
    csv << [title, (rank + 1), 'recommendations', nil, base_href, timestamp.to_s ]

    puts ['recommendations', (rank + 1), title].join("\t")  if is_verbose
  end
  sleep 1

  ## get list of things I've recently read (and put date of reading in timestamp column)
  list_page.search('.recentreads .story').each_with_index do |story, rank|
    a = story.search('h5 a').first
    title = a.text.gsub(/\s+/, ' ').strip
    base_href = a.attr('href').sub(/\?.+$/, '') # don't need the analytics parameters
    story_read_date = Time.parse(story.search('.dateline').text).strftime("%Y-%m-%d")
    csv << [title, (rank + 1), 'recently-read', nil, base_href, story_read_date]

    puts ['recently-read', story_read_date,  (rank + 1), title].join("\t")  if is_verbose
  end
end



### Don't really need Mechanize for this, but might as well just use the same call
### popular lists
lists = ['most-popular-viewed', 'most-popular-emailed', 'most-popular-blogged', 'most-popular-tweeted', 'most-popular-facebook'].product([1, 7, 30])
lists.each do |(list_name, period)|
  list_url = "http://www.nytimes.com/#{list_name}?period=#{period}"
  puts '', "-----------", list_url  if is_verbose if is_verbose
  agent.get(list_url) do |list_page|
    list_page.search('.story h3 a').each_with_index do |a, rank|
      title = a.text.gsub(/\s+/, ' ').strip
      base_href = a.attr('href').sub(/\?.+$/, '') # don't need the analytics parameters
      csv << [title, (rank + 1), list_name, period, base_href, timestamp.to_s ]

      puts [list_name, period, (rank + 1), title].join("\t")  if is_verbose
    end
    sleep 1
  end
end


# hell, let's get front page and mobile too
[["www", ['#top-news', '.c-column-top-span-region'], '.story-heading a'],
["mobile", ['#articles'], '.headlines li > a']
].each do |(cname, sections, selector)|
  agent.get("http://#{cname}.nytimes.com") do |page|
    sections.each do |section_name|
      page.search("#{section_name} #{selector}").each_with_index do |a, rank|
        list_name = "#{cname} #{section_name}"
        title = a.text.gsub(/\s+/, ' ').strip
        base_href = a.attr('href').sub(/\?.+$/, '') # don't need the analytics parameters
        csv << [title, (rank + 1), list_name, "", base_href, timestamp.to_s ]
        puts [list_name, (rank + 1), title].join("\t")  if is_verbose
      end
    end
  end
end


# Output of the program to terminal will look like:

# -----------
# http://www.nytimes.com/most-popular-tweeted?period=1
# most-popular-tweeted  1 1 Left Coast Rising
# most-popular-tweeted  1 2 Don’t Teach Math, Coach It
# most-popular-tweeted  1 3 Russia Steps Up Help for Rebels in Ukraine War
# most-popular-tweeted  1 4 Why Do Americans Stink at Math?
# most-popular-tweeted  1 5 U.S. Considering Refugee Status for Hondurans

sample-scrape-2014-07-29_23_27_27.csv