[AWS S3 client in Ruby]

aws_s3_bucket_resource.rb

# Create an instance of an AWS S3 resource (bucket)
class S3Bucket
  def self.fetch
    s3 = Aws::S3::Resource.new(
      Aws::Credentials.new(
        Rails.configuration.aws_uploader_access_key_id,
        Rails.configuration.aws_uploader_secret_access_key
      ),
    )
    s3.bucket(Rails.configuration.aws_bucket)
  end
end

bucket = S3Bucket.fetch

# Specify the S3 filepath (key) and content of the file you want to upload
s3_filepath = 'path/to/my_new_file.txt' # directories will be created automatically, if necessary
file_content = 'Hello, World!'

# Upload file content to S3
bucket.object(s3_filepath).put(body: file_content)

# Upload a file to S3
bucket.object(s3_filepath).upload_file(local_filepath)
#=> returns true if successful; throws Errno::ENOENT if local file doesn't exist

# Check if file exists in S3
file_exists = bucket.object(s3_filepath).exists?

# Read file content from S3
file_content = bucket.object(s3_filepath).get.body.read

# s3_bucket = S3Bucket.fetch
# s3_filepath = 'cooooooooool.txt'
# file_content = 'Wow, this is cool!'
# s3_bucket.object(s3_filepath).put(body: file_content)
# etag = s3_bucket.object(s3_filepath).get.etag
# last_modified = s3_bucket.object(s3_filepath).get.last_modified
# resp = s3_bucket.object(s3_filepath).get
# resp.body.read
# s3_bucket.object(s3_filepath).exists?

aws_s3_client.rb

# frozen_string_literal: true

require 'aws-sdk'

# AWS S3 client for uploading and downloading files
class S3Client
  attr_reader :bucket, :s3

  # The maximum number of times to attempt to upload or download a file
  DEFAULT_MAX_UPLOAD_ATTEMPTS = 20
  DEFAULT_MAX_DOWNLOAD_ATTEMPTS = 3
  private_constant :DEFAULT_MAX_UPLOAD_ATTEMPTS, :DEFAULT_MAX_DOWNLOAD_ATTEMPTS

  # The number of seconds to wait between attempts to upload or download a file
  DEFAULT_DELAY = 5
  private_constant :DEFAULT_DELAY

  # Set up the S3 client
  def initialize(bucket: Rails.configuration.aws_bucket,
                 region: Rails.configuration.aws_region,
                 access_key_id: Rails.configuration.aws_access_key_id,
                 secret_access_key: Rails.configuration.aws_secret_access_key)
    @bucket = bucket
    @s3 = Aws::S3::Client.new(region: region, access_key_id: access_key_id, secret_access_key: secret_access_key)
  end

  # Upload a file to S3
  #
  # If the directory does not exist, it (and all of it's parent directories) will be created.
  #
  # @param s3_file [String] the path to upload the file to, e.g. 'path/to/my_s3_file.txt'
  # @param body [String] the body of the file to upload (will take precedence if local_file also provided)
  # @param local_file [String] the path to the local file to upload (if body is not provided)
  # @param max_attempts [Integer] the maximum number of times to attempt to upload the file
  # @param delay [Integer] the number of seconds to wait between attempts to upload the file
  #
  # @return [Boolean] true if the file successfully uploaded, false otherwise
  def put_file(s3_file:, body: nil, local_file: nil, max_attempts: DEFAULT_MAX_UPLOAD_ATTEMPTS, delay: DEFAULT_DELAY)
    # FIXME: wait_until() will pass if the file already exists; should we remove the file first?
    #        Can you pass the checksum or a tag into wait_until()?
    body ||= File.read(local_file)
    # md5 = Digest::MD5.base64digest(body)
    s3.wait_until(:object_exists, bucket: bucket, key: s3_file) do |waiter|
      waiter.max_attempts = max_attempts
      waiter.delay = delay
      s3.put_object(bucket: bucket, key: s3_file, body: body)
    end
    true
  rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::PermanentRedirect
    Rails.logger.error "AWS S3 bucket '#{bucket}' does not exist."
    false
  rescue Aws::Waiters::Errors::WaiterFailed
    Rails.logger.error "File failed to upload to AWS S3 path '#{bucket}:#{s3_file}'."
    false
  end

  # Read a file from S3 into memory
  #
  # @param s3_file [String] the path to file that will be downloaded, e.g. 'path/to/my_s3_file.txt'
  #
  # @return [String, false] Downloaded file response or false if the file does not exist
  def read_file(s3_file)
    s3.get_object(bucket: bucket, key: s3_file).body.read
  rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::PermanentRedirect
    Rails.logger.error "AWS S3 bucket '#{bucket}' does not exist."
    false
  rescue Aws::S3::Errors::NoSuchKey
    Rails.logger.error "File '#{s3_file}' does not exist in AWS S3 bucket '#{bucket}'."
    false
  end

  # Check if a file exists in the S3 bucket
  #
  # @param s3_file [String] the path to the file to check, e.g. 'path/to/my_s3_file.txt'
  #
  # @return [Boolean] true if the file exists, false otherwise
  def file_exists?(s3_file)
    s3_file.chomp!('/')
    object_exists?(s3_file)
  end

  # Check if a directory exists in the S3 bucket
  #
  # @param s3_dir [String] the path to the directory to check, e.g. 'path/to/my_s3_dir/'
  #
  # @return [Boolean] true if the directory exists, false otherwise
  def dir_exists?(s3_dir)
    s3_dir += '/' unless s3_dir.end_with?('/')
    object_exists?(s3_dir)
  end

  private

  # Check if an object (i.e. file or directory) exists in the S3 bucket
  #
  # @param s3_object [String] the path to the object to check, e.g. 'path/to/my_s3_object.txt'
  #
  # @return [Boolean] true if the object exists, false otherwise
  def object_exists?(s3_object)
    s3.list_objects(bucket: bucket, prefix: s3_object.to_s).contents.map(&:key).any? { |k| k == s3_object }
  end
end

aws_stub_helper.rb

# File: spec/support/aws_stub_helper.rb

# frozen_string_literal: true

# Use this helper to stub AWS S3 requests in your tests. This is useful for testing
# methods that upload/download files to/from S3.
#
# To use this helper, add `:stub_aws` to your test:
#
#   RSpec.describe GenerateGooglePlacesReportJob, :stub_aws, type: :job do
#     # ...
#   end
#
# This will stub all AWS S3 requests, allowing you to test your methods without actually
# uploading files to S3.
#
# For more on AWS stubbing, see:
# - https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/ClientStubs.html
# - https://aws.amazon.com/blogs/developer/advanced-client-stubbing-in-the-aws-sdk-for-ruby-version-3/
module AwsStubHelper
  def stub_aws
    allow(Aws::S3::Resource).to receive(:new).and_return(S3::Resource.stub)
  end

  module S3
    module Resource
      class << self
        def stub # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
          resource = Aws::S3::Resource.new(
            Aws::Credentials.new(
              Rails.configuration.aws_uploader_access_key_id,
              Rails.configuration.aws_uploader_secret_access_key
            ),
            stub_responses: true
          )

          # Create a reports/ directory in the bucket by default
          bucket = { 'reports/' => { body: '' } }

          # Stubbing `Aws::S3::Client#get_object(...)` will also stub `Aws::S3::Resource#object(...).get`
          resource.client.stub_responses(
            :get_object,
            lambda do |context|
              obj = bucket[context.params[:key]]
              obj || 'NoSuchKey'
            end
          )

          # Stubbing `Aws::S3::Client#put_object(...)` will also stub `Aws::S3::Resource#object(...).put(...)`
          resource.client.stub_responses(
            :put_object,
            lambda do |context|
              params = context.params
              obj = bucket[params[:key]]

              cur_body = obj.present? ? obj[:body] : nil
              new_body = params[:body].to_s

              # If the new body is the same as the current body, keep the same ETag
              # This is essentially how it actually works in the real S3
              cur_etag = obj.present? ? obj[:etag] : ''
              new_etag = cur_body == new_body ? cur_etag : Digest::MD5.hexdigest(new_body + Time.now.to_s)

              bucket[params[:key]] = {
                body: new_body,
                accept_ranges: 'bytes',
                last_modified: Time.now, # this changes every time even if the body is the same
                content_length: new_body.length,
                etag: new_etag,
                content_type: params[:content_type] || '',
                server_side_encryption: 'AES256',
                metadata: params[:metadata] || {}
              }
              {}
            end
          )

          # Stubbing `Aws::S3::Client#head_object(key)` will also stub `Aws::S3::Resource#object(key).exists?`
          resource.client.stub_responses(
            :head_object,
            lambda do |context|
              obj = bucket[context.params[:key]]
              obj.present? ? { content_length: obj[:body].to_s.length } : nil
            end
          )

          resource.bucket(Rails.configuration.aws_bucket)
          resource
        end
      end
    end
  end
end

RSpec.configure do |config|
  config.include AwsStubHelper

  config.before(:each, :stub_aws) do
    stub_aws
  end
end