mechanize-sidekiq

gistfile1.txt

require 'mechanize'
require 'sidekiq'

class BackgroundWebCrawler < Mechanize

  include Sidekiq::Worker

  def perform(url)
    page = get(url)
    @links = page.links
    while link = links.shift
      page = click(link)
      visit(page)
    end
  end
 
 private
 
  def visit(page)
    puts page.title
    @links.concat(page.links)
  end
 end
 
 BackgroundWebCrawler.perform_async('http://example.com')