Copy contents of an S3 bucket to a another bucket using an EC2 instance and a simple Ruby script. Useful for transferring large amounts of data and will work across geographic regions.

copy-s3-bucket.rb

require 'rubygems'
require 'right_aws'

aws_access_key_id = 'your-access-key'
aws_secret_access_key = 'your-secret-key'
target_bucket = 'your-source-bucket'
destination_bucket = 'your-destination-bucket'

s3 = RightAws::S3Interface.new(aws_access_key_id, aws_secret_access_key)

copied_keys = Array.new
s3.incrementally_list_bucket(destination_bucket) do |key_set| 
  copied_keys << key_set[:contents].map{|k| k[:key]}.flatten 
end
copied_keys.flatten!

s3.incrementally_list_bucket(target_bucket) do |key_set|
  key_set[:contents].each do |key|
    key = key[:key]
    if copied_keys.include?(key)
      puts "#{destination_bucket} #{key} already exists. Skipping..."
    else
      puts "Copying #{target_bucket} #{key}"

      retries=0
      begin
        s3.copy(target_bucket, key, destination_bucket)
      rescue Exception => e
        puts "cannot copy key, #{e.inspect}\nretrying #{retries} out of 10 times..."
        retries += 1
        retry if retries <= 10
      end
    end
  end
end

ubuntu-stuff.sh

sudo add-apt-repository "deb http://archive.canonical.com/ lucid partner"
sudo apt-get update
sudo apt-get upgrade
sudo apt-get install ruby
sudo apt-get install rubygems
sudo apt-get install libopenssl-ruby
gem install right_aws

# create Ruby script (see copy-s3-bucket.rb above)

# run the script in the background
nohup ruby copy-s3-bucket.rb &

# watch the output
tail -f nohup.out

# for total bucket size, you can install and use s3cmd
sudo apt-get install s3cmd
# configure with s3 credentials
s3cmd --configure
s3cmd du s3://your.bucket