Crawl HTTP Proxy IPs

proxy_ip.rb

#!/usr/bin/env ruby

require "mechanize"
require 'logger'

logger = Logger.new(STDOUT)
TEST_URL = "http://baidu.com"

Entry = Struct.new(:host, :port) do
  attr_accessor :alive

  def to_s
    "#{host}:#{port}" 
  end
end

agent = Mechanize.new
agent.user_agent_alias = 'Windows IE 9'
agent.get 'http://cn-proxy.com/'

entries = agent.page.search(".entry-content table tbody tr").map do |tr|
  host = tr.children[0].text.strip
  port = tr.children[2].text.strip
  Entry.new(host, port)
end

entries.each do |entry|
  agent.set_proxy entry.host, entry.port
  begin
    Timeout.timeout(3) do
      agent.get TEST_URL
    end

    logger.debug "[ALIVE] #{entry}"
    entry.alive = true
  rescue
    logger.debug "[DEAD] #{entry}"
    entry.alive = false
  end 
end

puts "==[ALIVE]=="

entries.select{|e| e.alive}.each do |entry|
  puts entry.to_s
end