dump_qa.rb

desc 'download all s3 files for a given bucket'
task :download_s3 do
  s3 = AWS::S3::Client.new
  bucket_name = ENV['AWS_S3_BUCKET']
  FileUtils.mkdir_p('tmp/s3_dump')
  next_marker = ''
  begin
    resp = s3.list_objects bucket_name: bucket_name, marker: next_marker
    resp.contents.each do |f|
      dest = Rails.root.join("tmp/s3_dump", f.key)
      print "\33[2K\r Downloading: #{f.key} to #{dest}"
      FileUtils.mkdir_p File.dirname(dest)
      File.open(dest, "wb") do |file|
        file.write s3.get_object(bucket_name: bucket_name, key: f.key)[:data]
      end
    end
    next_marker = (resp.contents.last || {})[:key]
  end while next_marker.present?
end