fog-multipart.rb

#!/usr/bin/env ruby1.9.1
#
# Testing multipart uploads into s3
# Very basic script for testing how the functionality works 
#
# Takes a file, splits it up
# For each part get the base64 encoded md5 of the part
# Then run through the parts and upload them

# Refs:
# http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadInitiate.html
# http://docs.amazonwebservices.com/AWSEC2/latest/UserGuide/index.html?using-query-api.html <-- Query API auth
#

require 'fog'
require 'digest/md5'
require 'base64'
require 'fileutils'

key = 'AAAA'
secret = 'BBBB'
bucket = 'some_bucket'
region = 'eu-west-1'

# Setup connection
@stor = Fog::Storage.new(
  :provider => 'AWS',
  :aws_access_key_id => key,
  :aws_secret_access_key => secret,
  :region => region 
)


@stor.sync_clock

# Get object, split into 100M chunks
object_to_upload = '/tmp/linux-2.6.38.tar.bz2'

# Area to place the split file into
workdir = "/tmp/work/#{File.basename(object_to_upload)}/"
FileUtils.mkdir_p(workdir)

# Split the file into chunks
`split -C 10M -a 3 -d #{object_to_upload} #{workdir}`

# Map of the file_part => md5
parts = {}

# Get the MD5 of each file
Dir.entries(workdir).each do |file|
  next if file =~ /\.\./
  next if file =~ /\./

  md5 = Base64.encode64(Digest::MD5.file("#{workdir}/#{file}").digest).chomp!

  full_path = "#{workdir}#{file}"

  parts[full_path] = md5
end


# Initiate the upload and get the uploadid
multi_part_up = @stor.initiate_multipart_upload(bucket, object_to_upload[1..-1], { 'x-amz-acl' => 'private' } )
upload_id = multi_part_up.body["UploadId"]

part_number = 1
tags = []

parts.sort.each do |entry|
  puts "Got: #{entry[0]} - #{entry[1]}"

  @stor.reload

  File.open(entry[0]) do |file_part|
    part_upload = @stor.upload_part(bucket, object_to_upload[1..-1], upload_id, part_number, file_part, { 'Content-MD5' => entry[1] } )
    tags.push(part_upload.headers["ETag"])
  end

  part_number += 1

end


completed_upload = @stor.complete_multipart_upload(bucket, object_to_upload[1..-1], upload_id, tags)
p completed_upload