Last active
July 5, 2019 13:20
-
-
Save mattboldt/6052bac987c16b73563d4d6c56d7509b to your computer and use it in GitHub Desktop.
Update S3 object metadata of an entire bucket using the AWS Ruby SDK
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require 'aws-sdk' | |
require 'active_support' | |
require 'active_support/core_ext' | |
# http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Object.html#copy_to-instance_method | |
COPY_TO_OPTIONS = [:multipart_copy, :content_length, :copy_source_client, :copy_source_region, :acl, :cache_control, :content_disposition, :content_encoding, :content_language, :content_type, :copy_source_if_match, :copy_source_if_modified_since, :copy_source_if_none_match, :copy_source_if_unmodified_since, :expires, :grant_full_control, :grant_read, :grant_read_acp, :grant_write_acp, :metadata, :metadata_directive, :tagging_directive, :server_side_encryption, :storage_class, :website_redirect_location, :sse_customer_algorithm, :sse_customer_key, :sse_customer_key_md5, :ssekms_key_id, :copy_source_sse_customer_algorithm, :copy_source_sse_customer_key, :copy_source_sse_customer_key_md5, :request_payer, :tagging, :use_accelerate_endpoint] | |
Aws.config.update({ | |
region: 'us-east-1', | |
credentials: Aws::Credentials.new(AWS_ACCESS_KEY, AWS_SECRET_KEY) | |
}) | |
s3 = Aws::S3::Resource.new | |
bucket = s3.bucket('mattboldt-bucket') | |
objects_count = bucket.objects.count | |
current_object = 0 | |
bucket.objects.each do |object_summary| | |
# Get the object and all its metadata, permissions, etc | |
object = object_summary.get | |
# Copy to the same location | |
location = "#{bucket.name}/#{object_summary.key}" | |
# Build a new options object | |
options = {} | |
# Merge in the object's existing properties, but only keeping valid attributes for the copy_to method | |
existing_options = object.to_h.slice(*COPY_TO_OPTIONS) | |
options.merge!(existing_options) | |
# Add our new updates | |
options.merge!({ | |
acl: 'public-read', # private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read, bucket-owner-full-control | |
cache_control: 'public, max-age=60', | |
metadata_directive: 'REPLACE' | |
}) | |
# multipart_copy is necessary if the object is 5GB+ | |
if object.size >= 5_000_000_000 | |
options.merge!({multipart_copy: true}) | |
else | |
# Only used if multipart_copy is true | |
options.delete(:content_length) | |
end | |
begin | |
object_summary.copy_to(location, options) | |
current_object += 1 | |
puts "Copied #{current_object} of #{objects_count}" | |
rescue => e | |
puts "Excption Raised: #{e}" | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment