spider.rb

spider.rb

require 'rubygems'
require 'anemone'

count = 0

Anemone.crawl("http://www.immobilienscout24.de/immobiliensuche/wohnen-auf-zeit/nordrhein-westfalen.htm", :storage => Anemone::Storage.TokyoCabinet("/tmp/spidr.tch")) do |anemone|

  anemone.focus_crawl do |page|
    page.links.select do |link|
      link.request_uri =~ /(wohnen-auf-zeit)|(\/\d+$)/
    end
  end

  anemone.on_pages_like(/\/\d+$/) do |page|
    count += 1
    puts "scanning site number #{count} #{page.url}" if page
    puts page.doc.css('.is24-headline-type01f').text if page.doc
  end

end