AWS S3 read/write Benchmark

Gemfile

source :gemcutter
gem 'pit'
gem 'sauberia-aws-s3'

s3bench.rb

# s3bench.rb
#
# USAGE:
# $ bundle install
# $ ruby s3bench.rb [N]
#
require "rubygems"
require "benchmark"

require "bundler"
Bundler.setup
require "aws/s3"
require "pit"

include AWS::S3

CONFIG = Pit.get('s3bench', :require => {
  'access_key_id' => '',
  'secret_access_key' => '',
  'bucket_name' => ''
})

BUCKET_NAME = CONFIG['bucket_name']
N = (ARGV.shift || 1).to_i

def dummy_filename
  "/tmp/s3bench-dummy"
end

def filename(i)
  "s3bench-#{i}"
end

def local_filename(i)
  "/tmp/#{filename(i)}"
end

def remote_filename(i)
  "s3bench/#{filename(i)}"
end

def bench(title)
  puts "\nBenchmark: #{title} (#{N} times)"
  Benchmark.bm(7, ">total:", ">avg:") do |bm|
    results = []
    N.times do |i|
      results << bm.report(i.to_s) do
        yield i
      end
    end
    total = results.inject {|sum, t| sum+=t}
    [total, total/N]
  end
end

Base.establish_connection!(
  :access_key_id     => CONFIG['access_key_id'],
  :secret_access_key => CONFIG['secret_access_key'],
  :server => "#{BUCKET_NAME}.s3.amazonaws.com"
)

`dd if=/dev/zero of=#{dummy_filename} count=1 bs=5m`

bench("upload") do |i|
  S3Object.store(
    remote_filename(i),
    open(dummy_filename),
    BUCKET_NAME
  )
end

bench("download") do |i|
  open(local_filename(i), 'w') do |file|
    S3Object.stream(remote_filename(i), BUCKET_NAME) do |chunk|
      file.write chunk
    end
  end
  File.delete local_filename(i)
end

bench("exists?") do |i|
  S3Object.exists? remote_filename(i), BUCKET_NAME
end

bench("delete") do |i|
  S3Object.delete remote_filename(i), BUCKET_NAME
end

File.delete dummy_filename

From Yahoo! BB mobilepoint to S3(southeast)

Benchmark: upload (10 times)
             user     system      total        real
>total:  0.220000   0.410000   0.630000 (508.744421)
>avg:    0.022000   0.041000   0.063000 ( 50.874442)

Benchmark: download (10 times)
             user     system      total        real
>total:  6.770000   1.250000   8.020000 ( 84.046844)
>avg:    0.677000   0.125000   0.802000 (  8.404684)

Benchmark: exists? (10 times)
             user     system      total        real
>total:  0.020000   0.010000   0.030000 (  3.187673)
>avg:    0.002000   0.001000   0.003000 (  0.318767)

Benchmark: delete (10 times)
             user     system      total        real
>total:  0.010000   0.000000   0.010000 (  2.162521)
>avg:    0.001000   0.000000   0.001000 (  0.216252)

From EC2(southeast) to S3(southeast)

Benchmark: upload (10 times)
             user     system      total        real
>total:  0.000000   0.000000   0.000000 (  3.498857)
>avg:    0.000000   0.000000   0.000000 (  0.349886)

Benchmark: download (10 times)
             user     system      total        real
>total:  0.590000   0.080000   0.670000 (  2.756278)
>avg:    0.059000   0.008000   0.067000 (  0.275628)

Benchmark: exists? (10 times)
             user     system      total        real
>total:  0.010000   0.000000   0.010000 (  0.125816)
>avg:    0.001000   0.000000   0.001000 (  0.012582)

Benchmark: delete (10 times)
             user     system      total        real
>total:  0.000000   0.000000   0.000000 (  0.139655)
>avg:    0.000000   0.000000   0.000000 (  0.013965)

From Flet's Hikari to S3(southeast)

Benchmark: upload (10 times)
             user     system      total        real
>total:  0.230000   0.410000   0.640000 ( 77.704390)
>avg:    0.023000   0.041000   0.064000 (  7.770439)

Benchmark: download (10 times)
             user     system      total        real
>total:  9.350000   1.570000  10.920000 ( 57.733213)
>avg:    0.935000   0.157000   1.092000 (  5.773321)

Benchmark: exists? (10 times)
             user     system      total        real
>total:  0.010000   0.000000   0.010000 (  1.853019)
>avg:    0.001000   0.000000   0.001000 (  0.185302)

Benchmark: delete (10 times)
             user     system      total        real
>total:  0.020000   0.010000   0.030000 (  1.876039)
>avg:    0.002000   0.001000   0.003000 (  0.187604)