Easily loop through AR database records in chunks. Great for migrations.

gistfile1.txt

# If you're doing migrations or work on large datasets,
# you don't want to do a Record.find(:all).each.
#
# Why you ask? Well, unless you have gobs of system memory, your system
# will be paging like gangbusters when it starts loading the entire db table
# into memory as AR objects.
#
# To use this just drop this file into the /lib dir of your Rails project and
# you'll be able to use it in any migration
#

module RecordsInChunks

  # Loop through all records for a specified ActiveRecord model
  # prints out progress to STDOUT
  #
  # Usage:
  #
  #   RecordsInChunks.in_chunks(Project, 100) { |project| project.do_stuff! }
  #
  def self.in_chunks(klass, chunk_size = 25, &block)
    count = klass.count
    chunks = (count / chunk_size.to_f).ceil
    offsets = []
    chunks.times { |i| offsets << i * chunk_size }
    
    puts ">>> Looping through #{count} #{klass}s ... "
    
    for offset in offsets
      print "     > #{offset} - #{offset + chunk_size} / #{count} ... "
      STDOUT.flush
      
      klass.find(:all, :limit => chunk_size, :offset => offset).each do |o|
        yield(o)
      end
      
      puts "done"
    end
    
    puts "<<< Done looping through #{count} #{klass}s"
  end
 
end