Lazy sequences in Ruby

gistfile1.txt

lazy_expensive_sequence = Enumerator.new do |y|
  offset = 0
  amount_to_retrieve = 50
  loop do
    ## This may not be memory safe if we try to retrieve all at once, so we safely retrieving an amount that we know will fit in memory
    retrieve_memory_hogs(offset, amount_to_retrieve).each do |i|
      y << i
    end
    offset += amount_to_retrieve
  end
end

## realize the sequence without consuming too much memory
lazy_expensive_sequence.each do |i|
  ## I never have to worry about overloading memory
  ## I never have to worry about the logic behind how I chunk my memory
  ## I just consume
end


## Lazy composition
def select_map(collection, filter, transform)
  Enumerator.new do |y|
    collection.each do |i|
      y << transform.call(i) if filter.call(i)
    end
  end
end


## Concurrency via threads
def retrieve_sources(website_urls)
  Enumerator.new do |y|
    website_urls.each do |url|
      Thread.new do
        y << get_source(url)
      end
    end
  end
end

## Consume the concurrent lazy sequence
## the consumer doesn't know or care whether it's using threads. It just gets yielded results
retrieve_sources(%w(github.com google.com facebook.com)).each do |source|
  analyze(source)
end