Ruby File Streaming - line by line

rfs.rb

require "uri"
require "net/http"

class RemoteFileStream
  def initialize(url, headers: {})
    @uri = URI(url)
    @headers = headers
  end

  def each_line(&block)
    Net::HTTP.start(@uri.host, @uri.port, use_ssl: @uri.scheme == 'https') do |http|
      req = Net::HTTP::Get.new(@uri.request_uri)
      @headers.each { |k,v| req[k] = v }

      http.request(req) do |response|
        remainings = ""
        response.read_body do |chunk|
          chunk = remainings + chunk

          i = chunk.rindex("\n")
          data = chunk[0..i]
          remainings = chunk[i+1..-1]

          each_chunk(data, &block)
        end
        each_chunk(remainings, &block)
      end
    end
  end

  private
  def each_chunk(chunk, &block)
    chunk.split("\n").each { |line| yield(line) }
  end
end


require "csv"
class RemoteCsvStream
  def initialize(remote_file_stream)
    @remote_file_stream = remote_file_stream
  end

  def each_row(&block)
    @remote_file_stream.each_line do |line|
      row = CSV.parse(line)[0]

      if !@headers
        @headers = row.map(&:to_sym)
        next
      end

      yield(Hash[@headers.zip(row)])
    end
  end
end

# ---------------------------------

uri = ARGV.fetch(0) { puts("must provide url as first parameter"); exit() }
token = ARGV.fetch(1) { puts("must provide access token as second parameter"); exit() }

rfs = RemoteFileStream.new(uri, headers: { "Authorization" => "Bearer #{token}" })
csv = RemoteCsvStream.new(rfs)
csv.each_row do |row|
  printf("[%s] %f\n", row[:channel_id], row[:estimated_partner_revenue])
end