hyperfine benchmark s3 transfer

app.rb

require 'fog/aws'

BUCKET_NAME = ENV['BUCKET_NAME']
SOURCE_FILE_PATH = ENV['SOURCE_FILE_PATH']
TARGET_DIRECTORY = ENV['TARGET_DIRECTORY']
STORAGE_REGION = ENV["STORAGE_REGION"] || 'eu-central-1'
CHUNK_SIZE = ENV['CHUNK_SIZE'].to_i || 104_857_600
CONCURRENCY = ENV['CONCURRENCY'].to_i || 10

class S3Backup
  def initialize
    @connection = Fog::Storage.new(
      provider: 'AWS',
      use_iam_profile: false,
      aws_access_key_id: ENV['AWS_ACCESS_KEY_ID'],
      aws_secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
      aws_session_token: ENV['AWS_SESSION_TOKEN'],
      aws_credentials_expire_at: ENV['AWS_CREDENTIAL_EXPIRATION'],
      region: STORAGE_REGION,
    )
    @directory = @connection.directories.get(BUCKET_NAME) || @connection.directories.create(key: BUCKET_NAME)
  end

  def upload
    file = File.basename(SOURCE_FILE_PATH)
    target_path = File.join(TARGET_DIRECTORY, file)

    File.open(SOURCE_FILE_PATH, 'r') do |local_file|
      options = {
        key: target_path,
        body: local_file,
        public: false,
        multipart_chunk_size: CHUNK_SIZE,
        concurrency: CONCURRENCY,
      }
      @directory.files.create(options)
    end

    puts 'Upload completed successfully.'
  end
end

backup = S3Backup.new
backup.upload

Gemfile

source 'https://rubygems.org'

gem 'fog-aws', '~> 3.26'

repro.sh

#!/usr/bin/env bash

if [[ -z "$AWS_PROFILE" ]]; then
  echo "Error: please set AWS_PROFILE first!"
  exit 1
fi

if [[ -z "$AWS_REGION" ]]; then
  echo "Error: please set AWS_REGION first!"
  exit 1
fi

if [[ -z "$GEM_HOME" ]]; then
  echo "Error: please set GEM_HOME first!"
  exit 1
fi

commands_sanity=("ruby" "bundle" "aws" "dd" "hyperfine")

for cmd_var in "${commands_sanity[@]}"; do
  if ! command -v "$cmd_var" &> /dev/null ; then
    echo "Error: '$cmd_var' is not installed!"
    exit 1
  fi
done

bundle install

TARGET_ACCOUNT="$(aws sts get-caller-identity --output text --query 'Account')"
BUCKET_NAME="transfer-test-in-${TARGET_ACCOUNT}"

if ! aws s3api head-bucket --bucket "$BUCKET_NAME" 2>/dev/null ; then
  aws s3api create-bucket --bucket "$BUCKET_NAME" \
    --create-bucket-configuration LocationConstraint="$AWS_REGION"
  aws s3api put-bucket-encryption --bucket "$BUCKET_NAME" \
    --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"}}]}'
fi

SOURCE_FILE_PATH="${PWD}/file.blob"

export BUCKET_NAME="$BUCKET_NAME" SOURCE_FILE_PATH="$SOURCE_FILE_PATH" TARGET_DIRECTORY="transfer" STORAGE_REGION="$AWS_REGION"

eval $(aws configure export-credentials --format env)

export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_CREDENTIAL_EXPIRATION

hyperfine \
    --prepare "aws s3 rm s3://${BUCKET_NAME}/${TARGET_DIRECTORY}/file.blob ; dd if=/dev/urandom bs=1024 count=10000000 of=${SOURCE_FILE_PATH} conv=notrunc status=none" \
    --cleanup "rm -f ${SOURCE_FILE_PATH}" \
    --conclude "aws s3 rm s3://${BUCKET_NAME}/${TARGET_DIRECTORY}/file.blob" \
    --min-runs 5 \
    --parameter-list test_chunk_size 5242880,10485760,104857600,1073741824 \
    --parameter-list concurrency 1,5,10 \
    --command-name 'measure-transfer-rate-fog-5mb-1' \
    --command-name 'measure-transfer-rate-awscli-5mb-1' \
    --command-name 'measure-transfer-rate-fog-10mb-1' \
    --command-name 'measure-transfer-rate-awscli-10mb-1' \
    --command-name 'measure-transfer-rate-fog-100mb-1' \
    --command-name 'measure-transfer-rate-awscli-100mb-1' \
    --command-name 'measure-transfer-rate-fog-1000mb-1' \
    --command-name 'measure-transfer-rate-awscli-1000mb-1' \
    --command-name 'measure-transfer-rate-fog-5mb-5' \
    --command-name 'measure-transfer-rate-awscli-5mb-5' \
    --command-name 'measure-transfer-rate-fog-10mb-5' \
    --command-name 'measure-transfer-rate-awscli-10mb-5' \
    --command-name 'measure-transfer-rate-fog-100mb-5' \
    --command-name 'measure-transfer-rate-awscli-100mb-5' \
    --command-name 'measure-transfer-rate-fog-1000mb-5' \
    --command-name 'measure-transfer-rate-awscli-1000mb-5' \
    --command-name 'measure-transfer-rate-fog-5mb-10' \
    --command-name 'measure-transfer-rate-awscli-5mb-10' \
    --command-name 'measure-transfer-rate-fog-10mb-10' \
    --command-name 'measure-transfer-rate-awscli-10mb-10' \
    --command-name 'measure-transfer-rate-fog-100mb-10' \
    --command-name 'measure-transfer-rate-awscli-100mb-10' \
    --command-name 'measure-transfer-rate-fog-1000mb-10' \
    --command-name 'measure-transfer-rate-awscli-1000mb-10' \
        'CHUNK_SIZE={test_chunk_size} CONCURRENCY={concurrency} ruby app.rb' \
        "aws configure set s3.multipart_chunksize {test_chunk_size} ; \
         aws configure set s3.max_concurrent_requests {concurrency} ; \
         aws s3 cp ${SOURCE_FILE_PATH} s3://${BUCKET_NAME}/${TARGET_DIRECTORY}/file.blob" \
    --export-markdown benchmark.md

# hyperfine \
#     --show-output \
#     --runs 1 \
#     --parameter-list test_chunk_size 5242880,10485760,104857600,1073741824 \
#     --parameter-list concurrency 1,5,10 \
#         'echo CHUNK_SIZE={test_chunk_size} CONCURRENCY={concurrency} command 1 ; sleep 1 ; exit 0' \
#         'echo CHUNK_SIZE={test_chunk_size} CONCURRENCY={concurrency} command 2 ; sleep 1 ; exit 0'

$ curl -fsSLO https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine-v1.19.0-x86_64-unknown-linux-gnu.tar.gz
$ sudo tar -zxf hyperfine-v1.19.0-x86_64-unknown-linux-gnu.tar.gz -C /usr/local/bin/ --strip-components 1 hyperfine-v1.19.0-x86_64-unknown-linux-gnu/hyperfine
$ sudo chown root:root /usr/local/bin/hyperfine
$ sudo chmod +x /usr/local/bin/hyperfine
$ sudo dnf group install -y "Development Tools"
$ sudo dnf install -y ruby ruby-devel
$ touch ~/.aws/config
$ cat <<'EOF' >> ~/.aws/config
[profile development]
credential_source = Ec2InstanceMetadata
region = eu-central-1
EOF
$ export AWS_PROFILE='development'
$ export AWS_REGION='eu-central-1'
$ mkdir gems
$ export GEM_HOME="$(pwd)/gems"
$ curl -fsSLO https://gist.github.com/avoidik/9ce10e7ead1fe137af77a509828b62df/raw/fb301f70d95ac2b4a409a3ff64ebe7359086cb39/app.rb
$ curl -fsSLO https://gist.github.com/avoidik/9ce10e7ead1fe137af77a509828b62df/raw/fb301f70d95ac2b4a409a3ff64ebe7359086cb39/Gemfile
$ curl -fsSLO https://gist.github.com/avoidik/9ce10e7ead1fe137af77a509828b62df/raw/fb301f70d95ac2b4a409a3ff64ebe7359086cb39/repro.sh
$ bash repro.sh

Based on amount of synthetic data generated by dd (bs= x count=) this performance test may take more time than AWS STS credentials lifespan is. That's why you may find this test failing with something like:

<Error><Code>ExpiredToken</Code><Message>The provided token has expired.</Message>

Or:

Credentials were refreshed, but the refreshed credentials are still expired.

If this is the case, change files as follows:

# app.rb -- rely on the EC2 instance profile IAM role, remove static credentials
    @connection = Fog::Storage.new(
      provider: 'AWS',
      use_iam_profile: true,
      aws_credentials_refresh_threshold_seconds: 420,
      region: STORAGE_REGION,
    )

# repro.sh -- comment out these two lines
# eval $(aws configure export-credentials --format env)
# export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_CREDENTIAL_EXPIRATION