presidential speech markov model

speeches.rb

require 'open-uri'
require 'nokogiri'
require 'marky_markov'

markov = MarkyMarkov::Dictionary.new('dictionary')

page = Nokogiri::HTML(open("http://millercenter.org/president/speeches"))

speech_urls = []

page.css('a').each do |link|
  href = link.attributes["href"].to_s
  if href.include?("speeches/detail") and !speech_urls.include?(href)
    speech_urls << href
  end
end

speech_urls.each do |href|
  url = "http://millercenter.org/president/#{href}"
  puts "opening #{url}"
  speech_page = Nokogiri::HTML(open(url))
  markov.parse_string speech_page.css('#transcript').text
  markov.save_dictionary!
end

puts markov.generate_20_words