Example of batching requests using em-http-request

batch_em_http_example.rb

#!/usr/bin/env ruby

# This is an alternative to using EM::Iterator. It allows us to use
# MultiRequest but still limit the number of concurrent requests that
# are created. 

require 'rubygems'
require 'eventmachine'
require 'em-http-request'

class Batch < Array
  def each(batch_size, &block)
    loops = (size().to_f / batch_size).ceil
    (0..loops-1).each do |l|
      batch = slice(l*batch_size, batch_size)
      yield(batch)
    end
  end

  def each_with_index(batch_size, &block)
    c = 0
    each(batch_size) do |batch|
      yield(batch.to_a, c)
      c += batch_size
    end
  end
end

uris = [
  "http://www.google.com",
  "http://www.google.com",
  "http://www.google.com",
  "http://www.google.com",
  "http://www.google.com",
  "http://www.google.com",
]
threads = 4

results = []
Batch.new(uris).each_with_index(threads) do |batch, batch_pad|

  EventMachine.run do
    multi = EventMachine::MultiRequest.new

    batch.each_with_index do |uri, i|
      multi.add (i+batch_pad), EventMachine::HttpRequest.new(uri).get
    end

    multi.callback do
      puts "callback #{batch_pad}"
      batch.each_with_index do |uri, i|
        results << multi.responses[:callback][(i+batch_pad)].response
      end
      EventMachine.stop
    end
  end

end

puts "em run finished"
puts results.size

Thanks! Really useful stuff!