Grabs all the text and <a> tags from a web-page

text_and_links.rb

#!/usr/bin/env ruby

require 'nokogiri'
require 'open-uri'
require 'cgi'
require 'uri'

unless ARGV.length == 1
  STDERR.puts "usage: ./text_and_links URL"
  exit -1
end

IGNORE_TAGS = ['script', 'style']

def text_and_links(start,ignore=IGNORE_TAGS)
  buffer = []

  iterate = lambda { |root|
    root.children.each { |node|
      if node.text?
        buffer << node.to_html.strip
      elsif (node.name == 'a' && node['href'])
        href = URI.escape(node['href'])
        text = CGI.escapeHTML(node.inner_text)

        buffer << "<a href=#{href.dump}>#{text}</a>"
      elsif !(ignore.include?(node.name))
        iterate.call(node)
      end
    }
  }

  iterate.call(start)
  return buffer.join(' ')
end

doc = Nokogiri::HTML(open(ARGV[0]))
puts text_and_links(doc.at('body'))