link_checker.rb

require 'rubygems'
require 'pp'
require 'mechanize'

class LinkChecker
  PAGES_MAX = 1000

  def initialize(param=[])
    @crawled = {}
    @queue = []
    @queue_index = nil
    @queue_is_active = false
    @agent = WWW::Mechanize.new
    @whitelist = param[:whitelist] || ['.*']
  end

  def start_crawl(start_url)
    @queue.push start_url
    i = 0
    while (url = @queue.pop) && i<PAGES_MAX
      crawl_url url
      i += 1
    end
    @crawled
  end

private

  def crawl_url(url)
    page_links = []
    begin
      @agent.get url
      code = @agent.page.code

      unless match_whitelist(url)
        STDERR.puts "#{code} (not_white) #{url}"
        @crawled[url] = {:result=>code}
        return
      end

      links = @agent.page.links
      links.each do |link|
        next unless link.href
        link_url = (URI.parse(url) + URI.parse(link.href)).to_s
        page_links.push link_url
        if url != link_url &&
          [email protected]?(link_url) &&
          [email protected]?(link_url)

          @queue.push link_url
        end
      end
      STDERR.puts "#{code} (ok) #{url}"
      @crawled[url] = {:result=>code, :links=>page_links}
    rescue Object=>e
      STDERR.puts "#{code} (err) #{url}"
      @crawled[url] = {:result=>code, :error=>e}
    end
  end

  def match_whitelist(url)
    @whitelist.each do |white|
      return true if url=‾/#{white}/
    end
    false
  end

end

c = LinkChecker.new :whitelist=>['^http://tsuyabu¥.in/']
res = c.start_crawl 'http://tsuyabu.in/'
error_count = 0
res.each{|k,v| error_count+=1 if v[:result]!='200'}
STDERR.puts "total #{error_count} errors."

puts res.to_yaml