Streaming HTTP response body in Ruby, exposing an IO-like response object.

streaming_ruby_http_response.rb

require 'uri'
require 'net/protocol'
require 'byebug'

class Request

  VERBS = {
    get: 'GET'
  }

  attr_reader :uri

  def initialize(uri)
    @uri = uri.is_a?(String) ? URI(uri) : uri
  end

  def run(verb, path, headers = {}, &block)
    socket = TCPSocket.open(uri.host, uri.port, nil, nil)
    socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)

    lines = ["#{VERBS.fetch(verb.to_sym, 'GET')} #{path} HTTP/1.0"]
    lines << "Host: #{uri.host}\r\n"

    heads = headers.map do |k, v|
      [k, v].join(': ')
    end

    if heads.any?
      lines += heads
    else
      lines << "\r\n"
    end

    socket.write lines.join("\r\n")

    # socket.close_write

    response = Response.new(socket)
    if block_given?
      yield response
      response.close
    else
      response
    end
  end
end

class Response
  def initialize(io)
    @io = io
    @buf = ''
    @protocol_line = io.readline
    @headers = read_headers
  end

  def code
    @code ||= prot_segments[1].to_s.to_i
  end

  def message
    @message ||= prot_segments.last
  end

  SEPARATOR = "\n".freeze
  CHUNK_SIZE = 1024 * 16

  def read(*args)
    io.read *args
  end

  def readline
    begin
      until idx = buf.index(SEPARATOR)
        buf << io.read_nonblock(CHUNK_SIZE)
      end
      consume_buffer idx + SEPARATOR.size
    rescue EOFError
     consume_buffer buf.size
    rescue IO::WaitReadable
      if IO.select([io], nil, nil, nil)
        retry
      else
        raise Net::ReadTimeout
      end
    end
  end

  def close
    io.close unless io.closed?
  end

  def closed?
    io.closed?
  end

  private
  attr_reader :io, :buf

  def consume_buffer(len)
    buf.slice!(0, len)
  end

  def prot_segments
    @prot_segments ||= protocol_line.split(/\s+/)
  end

  def read_headers
    heads = {}
    each_response_header do |k, v|
      heads[k] = v
    end

    heads
  end

  def each_response_header
    key = value = nil
    while true
      line = readline.sub(/\s+\z/, '')
      break if line.empty?
      if line[0] == ?\s or line[0] == ?\t and value
        value << ' ' unless value.empty?
        value << line.strip
      else
        yield key, value if key
        key, value = line.strip.split(/\s*:\s*/, 2)
        raise 'wrong header line format' if value.nil?
      end
    end
    yield key, value if key
  end
end

# Example
Request.new('http://localhost:8000').run(:get, '/countries.xml') do |r|
  f = File.new('./target.xml', 'w')
  IO.copy_stream r, f
end