EventMachine async URL fetcher

fiber_pool.rb

# https://github.com/betamatt/fiber-test/blob/master/lib/fiber_pool.rb

require 'fiber'

class FiberPool
  def initialize(max)
    @max = max
    @fibers = []
    @queue = []
    
    max.times do 
      fiber = Fiber.new do |value, block|
        loop do
          block.call(value)
          
          if @queue.empty?
            @fibers << Fiber.current
            
            break if @done
            block = Fiber.yield
          else
            value, block = @queue.shift
          end          
        end
        
        teardown
      end
      
      @fibers << fiber
    end
  end

  def finish(parent)
    @done = true
    @parent = parent
  end
  
  def enqueue(value, &block)
    if @fibers.empty?
      @queue << [value, block]
    else
      fiber = @fibers.shift
      fiber.resume([value, block])
    end
  end
  
  private
  
    def teardown
      @parent.resume if @fibers.size == @max
    end
end

Gemfile

source "https://rubygems.org"

gem 'eventmachine'
gem 'em-http-request'

url_fetcher.rb

require 'eventmachine'
require 'em-http-request'
require './lib/fiber_pool'

URLS = [
  'http://example.com/',
  'https://www.cloudflare.com/robots2.txt', # 404 - not found
  'https://rubygems.org/gems/em-http-request/versions.atom',
  'http://twitter.com/robots.txt',
]

def url_handler(url)
  fiber = Fiber.current

  http = EventMachine::HttpRequest.new(url).get
  http.callback { fiber.resume(http) }
  http.errback  { p 'Uh oh'; fiber.resume(http) }

  Fiber.yield

  puts "#{http.response_header.status.inspect} #{url}"
  if http.response_header.status == 200
    p http.response[0,100]
  else
    p http.response_header
  end
  puts '-'*20
end

def main
  pool = FiberPool.new(3)

  EventMachine.run do
    URLS.each do |url|
      pool.enqueue(url) do |url|
        url_handler(url)
      end
    end

    # Need a notification when all of the items on the queue are done processing
    Fiber.new do 
      pool.finish(Fiber.current) 
      Fiber.yield # wait for callback
      EM.stop
    end.resume
  end
end

main