Rake task to copy assets between S3 buckets. Requires the aws-s3 gem (http://amazon.rubyforge.org/)

sync.rake

# add a new "put_copy" method to the Amazon client's S3Object class
# to enable copying an object from 1 bucket to another
# http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html
class AWS::S3::S3Object
  def self.put_copy(source_key, target_key, source_bucket, target_bucket, options = {})
    original = open(url_for(source_key, source_bucket))
    default_options = { :content_type => original.content_type }
    store(target_key, original, target_bucket, default_options.merge(options))
    acl(target_key, target_bucket, acl(source_key, source_bucket))
  end
end


namespace :s3 do
  desc 'Update an S3 bucket with the latest assets from the production bucket.'
  task :sync, [:bucket] => :settings do |t, args|
    source_bucket_name = '...'
    
    target_bucket_name = case args.bucket
      when 'development' then 'development-bucket'
      when 'staging'     then 'staging-bucket'
    else
      abort "\nUsage: rake s3:sync[env]\n\n"
    end

    puts "\nCopying assets from #{source_bucket_name} to #{target_bucket_name}...\n\n"
    
    AWS::S3::Base.establish_connection!(
      :access_key_id     => Settings.aws[:access_key_id],
      :secret_access_key => Settings.aws[:secret_access_key]
    )

    started = Time.now
    copied = 0
    skipped = 0
    marker = nil
    
    source_bucket = AWS::S3::Bucket.find(source_bucket_name)
    
    while true do
      # get the next group of objects in the source bucket
      objects = source_bucket.objects(:prefix => 'images', :marker => marker)
      break if objects.empty?
    
      objects.each do |source|
        key = source.key
        
        # rescue from AWS::S3::NoSuchKey error and set target to nil so it gets copied
        target = AWS::S3::S3Object.find(key, target_bucket_name) rescue nil
    
        if target.nil? || source.etag != target.etag
          # Retry the put_copy method 3 times to work around the "403
          # Forbidden" error intermittently received.
          retries = 3
          begin
            AWS::S3::S3Object.put_copy(key, key, source_bucket_name, target_bucket_name)
          rescue Exception => e
            puts "An exception occurred: #{e}"
            if retries > 0
              retries -= 1
              puts "Retrying..."
              retry
            end
          end
          puts "copy #{key}"
          copied += 1
        else
          puts "skip #{key}"
          skipped += 1
        end
      end
      
      # set marker so next iteration knows where to start
      marker = objects.last.key
    end
    
    elapsed = ((Time.now - started) / 60 * 100).to_i.to_f / 100

    puts "\nCompleted in #{elapsed} minutes"
    puts "#{skipped} files skipped"
    puts "#{copied} files copied\n\n"
  end
end