s3_bucket_clone.rb

#!/usr/bin/env ruby
## Taken from: http://snippets.dzone.com/posts/show/4935
#
# Usage:
# a = AmazonS3Asset.new
# a.copy_over_bucket("myapp_production", "myapp_production")

require 'aws/s3'
require 'mechanize'
require 'logger'
require 'yaml'

class AmazonS3Asset

  include AWS::S3
  CONFIG = YAML.load_file File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 's3.yml'))
  S3ID  = CONFIG['aws_access_key']
  S3KEY = CONFIG['aws_secret_access_key']
  
  def initialize
    puts "connecting..."
    AWS::S3::Base.establish_connection!(
      :access_key_id     => S3ID,
      :secret_access_key => S3KEY
    )
  end

  def delete_key(bucket, key)
    if exists?(bucket, key) 
      S3Object.delete key, bucket
    end
  end
  
  def empty_bucket(bucket)
    each_bucket_key(bucket) do |k|
      puts "deleting #{bucket}: #{k}"
      delete_key(bucket,k)
    end
  end
  
  def each_bucket_key(bucket_name, &block)
    marker_str = ""
    bucket = Bucket.find(bucket_name)
    begin
      objects = bucket.objects(:marker => marker_str)
      objects.each do |object|
        yield object.key
      end
      marker_str = objects.last.key unless objects.empty?
    end while not objects.empty?
  end

  def copy_over_bucket(from_bucket, to_bucket)
    puts "Replacing #{to_bucket} with contents of #{from_bucket}"
    # delete to_bucket
    empty_bucket(to_bucket)
    each_bucket_key(from_bucket) do |key|
      copy_between_buckets(from_bucket, to_bucket, key)
    end
  end
  
  def copy_between_buckets(from_bucket, to_bucket, from_key, to_key = nil)
    if exists?(from_bucket, from_key)
      to_key = from_key if to_key.nil?
      puts "Copying #{from_bucket}.#{from_key} to #{to_bucket}.#{to_key}"
      url = "http://s3.amazonaws.com/#{from_bucket}/#{from_key}"
      filename = download(url)
      store_file(to_bucket,to_key,filename)
      File.delete(filename)
      return "http://s3.amazonaws.com/#{to_bucket}/#{to_key}"
    else
      puts "#{from_bucket}.#{from_key} didn't exist"
      return nil
    end
  end

  def store_file(bucket, key, filename)
     puts "Storing #{filename} in #{bucket}.#{key}"
     S3Object.store(
      key,
      File.open(filename),
      bucket,
      :access => :public_read
      )
  end

  def download(url, save_as = nil)
    if save_as.nil?
      tmp_dir = '/tmp/amazon_s3_temp'
      Dir.mkdir(tmp_dir) unless File.exists?(tmp_dir)
      save_as = File.join(tmp_dir, File.basename(url))
    end
    begin
      puts "Saving #{url} to #{save_as}"
      agent = Mechanize.new
      img = agent.get(url)
      img.save_as(save_as)
      return save_as
    rescue => e
      puts e
      raise "Failed on " + url + "  " + save_as
    end
  end

  def exists?(bucket,key)
    begin
      res = S3Object.find key, bucket
    rescue 
      res = nil
    end
    return !res.nil?
  end
      
end