xmlsitemapcrawler.rb

require 'rubygems'
require "net/http"
require "net/https"
require 'nokogiri'

XML = Nokogiri::HTML open "sitemap.xml"
links = XML.css "loc"

links.each do |link|
  uri = URI link.text

  Net::HTTP.start uri.host, uri.port, :use_ssl => false, :verify_mode => OpenSSL::SSL::VERIFY_NONE do |http|
    request = Net::HTTP::Get.new uri
    response = http.request request
    puts "Response headers: #{response.to_hash.inspect}"
  end

  sleep 1
end