Robots.txt Spider

require 'rubygems'
require 'active_record'
require 'robots'

dbconfig = YAML::load(File.open("db.yml"))
ActiveRecord::Base.establish_connection(dbconfig)
ActiveRecord::Base.logger = Logger.new(STDERR)

class DomainName < ActiveRecord::Base
end

robots = Robots.new "Kfx2 Spider"
domains = DomainName.find(:all)

domains.each do |domain|
  url = "http://"+domain.hostname
  begin
    if !robots.allowed?(url)
      puts domain.hostname
    end
  rescue
      print "An error occurred: ",$!, "\n"
  end
end