gistfile1.rb

# Example of chunking an operation that would use too much memory
# Grit is a git repository library 

all_commits = -> grit_repo {
  Enumerator.new do |y|
    total_commit_count = grit_repo.commit_count    
    # we can safely handle 500 commits in memory
    amount = 500
    offset = 0
    ((total_commit_count / amount) + 1).times do
      # Retrieve `amount` commits in memory, yield, garbage collect, retrieve 500 more...
      grit_repo.commits('master', amount, offset).each {|commit| y << commit }
      offset += amount
    end
  end
}

# Import all commits into database or something
# The consumer of the enumerator does not know that the operation is memory efficient. It just consumes like it is an array
all_commits_enum = all_commits.(Grit::Repo.new('/path/to/repository'))
all_commits_enum.each do |commit|
  import(commit)
end