Created
January 17, 2013 00:21
-
-
Save mindreframer/4552325 to your computer and use it in GitHub Desktop.
S3 downloader (signed URL)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# encoding : utf-8 | |
#!/usr/bin/env ruby | |
require 'openssl' | |
require 'digest/sha1' | |
require 'base64' | |
require 'date' | |
require 'uri' | |
class S3; end | |
class << S3 | |
def aws_access_key_id | |
"YOUR_ACCESS_KEY" | |
end | |
def aws_secret_access_key | |
"YOUR_SECRET_ACCESS_KEY" | |
end | |
def bucket | |
"YOUR_BUCKET" | |
end | |
end | |
module Aws | |
extend self | |
def signed_url(path, expire_date) | |
digest = OpenSSL::Digest::Digest.new('sha1') | |
can_string = "GET\n\n\n#{expire_date}\n/#{S3.bucket}/#{path}" | |
hmac = OpenSSL::HMAC.digest(digest, S3.aws_secret_access_key, can_string) | |
signature = URI.escape(Base64.encode64(hmac).strip).encode_signs | |
"https://#{S3.bucket}.s3.amazonaws.com/#{path}?AWSAccessKeyId=#{S3.aws_access_key_id}&Expires=#{expire_date}&Signature=#{signature}" | |
end | |
end | |
class String | |
def encode_signs | |
signs = {'+' => "%2B", '=' => "%3D", '?' => '%3F', '@' => '%40', | |
'$' => '%24', '&' => '%26', ',' => '%2C', '/' => '%2F', ':' => '%3A', | |
';' => '%3B', '?' => '%3F'} | |
signs.keys.each do |key| | |
self.gsub!(key, signs[key]) | |
end | |
self | |
end | |
end | |
class Downloader | |
def pathes | |
(01..36).to_a.map{|x| "06/backup-20130106.gz.#{x.to_s.rjust(2, '0')}"} | |
end | |
def date | |
@date ||= (Time.now + 10*60*60).to_i # expires in 10 hours | |
end | |
def signed_urls | |
pathes.map do |path| | |
signed_url = Aws.signed_url(path, date) | |
end | |
end | |
def filename(url) | |
File.basename(url.split('?').first) | |
end | |
def commands | |
cmds = signed_urls.map do |url| | |
fname = filename(url) | |
next if File.exists?(fname) and !File.exists?(fname + ".aria2") # download really finished | |
#cmd = %Q{wget --no-check-certificate -O=#{filename} "#{url}"} | |
cmd = %Q{aria2c -s 10 -o #{fname} "#{url}"} | |
end | |
(cmds - [nil]) | |
end | |
def run(should_run=false) | |
commands.each do |cmd| | |
if should_run | |
sh(cmd) | |
else | |
puts cmd | |
end | |
end | |
end | |
# simplified copy of rake`s sh | |
def sh(cmd) | |
puts cmd | |
IO.popen(cmd) do |pipe| | |
while str = pipe.gets | |
puts str | |
end | |
end | |
$?.success? | |
end | |
end | |
should_run = ARGV[0] ? true : false | |
Downloader.new.run(should_run) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment