-
-
Save johnjohndoe/4310160 to your computer and use it in GitHub Desktop.
Replace hard coded file name with environment variable.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env ruby | |
# An HTTP/HTTPS/FTP file downloader library/CLI based upon MiniPortile's | |
# HTTP implementation. | |
# | |
# Author: Jon Maken | |
# License: 3-clause BSD | |
# Revision: 2012-03-25 21:18:59 -0600 | |
require 'net/http' | |
require 'net/https' if RUBY_VERSION < '1.9' | |
require 'net/ftp' | |
require 'fileutils' | |
require 'tempfile' | |
class Downloader | |
@logger = STDOUT.binmode | |
@max_ca_verify_depth = 5 | |
def self.download_file(url, full_path, count = 3) | |
return if File.exist?(full_path) | |
uri = URI.parse(url) | |
case uri.scheme.downcase | |
when /ftp/ | |
ftp_download(uri, full_path) | |
when /http|https/ | |
http_download(url, full_path, count) | |
end | |
end | |
private | |
def self.message(text) | |
@logger.print text | |
@logger.flush | |
end | |
def self.output(text = '') | |
@logger.puts text | |
@logger.flush | |
end | |
def self.http_download(url, full_path, count) | |
filename = File.basename(full_path) | |
begin | |
uri = URI.parse(url) | |
if ENV['HTTP_PROXY'] | |
protocol, userinfo, proxy_host, proxy_port = URI::split(ENV['HTTP_PROXY']) | |
proxy_user, proxy_pass = userinfo.split(/:/) if userinfo | |
http = Net::HTTP.new(uri.host, uri.port, proxy_host, proxy_port, proxy_user, proxy_pass) | |
else | |
http = Net::HTTP.new(uri.host, uri.port) | |
end | |
if uri.scheme.downcase == 'https' | |
http.use_ssl = true | |
if ENV['CA_CERT_FILE'] | |
cert_file = ENV['CA_CERT_FILE'].dup | |
cert_file.gsub!(File::ALT_SEPARATOR, File::SEPARATOR) if File::ALT_SEPARATOR | |
end | |
if cert_file && File.exist?(cert_file) | |
http.ca_file = cert_file | |
http.verify_mode = OpenSSL::SSL::VERIFY_PEER | |
http.verify_depth = @max_ca_verify_depth | |
else | |
raise <<-EOT | |
To download using HTTPS you must first set the CA_CERT_FILE | |
environment variable to the path of a valid CA certificate file. | |
A file of bundled public CA certs may be downloaded from: | |
http://curl.haxx.se/ca/cacert.pem | |
EOT | |
end | |
end | |
http.request_get(uri.path) do |response| | |
case response | |
when Net::HTTPNotFound | |
output "404 - Not Found" | |
return false | |
when Net::HTTPClientError | |
output "Error: Client Error: #{response.inspect}" | |
return false | |
when Net::HTTPRedirection | |
raise "Too many redirections for the original URL, halting." if count <= 0 | |
url = response["location"] | |
return download_file(url, full_path, count - 1) | |
when Net::HTTPOK | |
temp_file = Tempfile.new("download-#{filename}") | |
temp_file.binmode | |
size = 0 | |
progress = 0 | |
total = response.header["Content-Length"].to_i | |
response.read_body do |chunk| | |
temp_file << chunk | |
size += chunk.size | |
new_progress = (size * 100) / total | |
unless new_progress == progress | |
message "\rDownloading %s (%3d%%) " % [filename, new_progress] | |
end | |
progress = new_progress | |
end | |
output | |
temp_file.close | |
File.unlink full_path if File.exists?(full_path) | |
FileUtils.mkdir_p File.dirname(full_path) | |
FileUtils.mv temp_file.path, full_path, :force => true | |
end | |
end | |
rescue Exception => e | |
File.unlink full_path if File.exists?(full_path) | |
output "ERROR: #{e.message}" | |
raise "Failed to download file" | |
end | |
end | |
def self.ftp_download(parsed_uri, full_path) | |
filename = File.basename(parsed_uri.path) | |
begin | |
temp_file = Tempfile.new("download-#{filename}") | |
temp_file.binmode | |
size = 0 | |
progress = 0 | |
# TODO add user/pw support | |
Net::FTP.open(parsed_uri.host) do |ftp| | |
ftp.passive = true | |
ftp.login | |
ftp.chdir(File.dirname(parsed_uri.path)) | |
total = ftp.size(filename) | |
ftp.getbinaryfile(filename, nil, 8192) do |chunk| | |
temp_file << chunk | |
size += chunk.size | |
new_progress = (size * 100) / total | |
unless new_progress == progress | |
message "\rDownloading %s (%3d%%) " % [filename, new_progress] | |
end | |
progress = new_progress | |
end | |
end | |
output | |
temp_file.close | |
File.unlink full_path if File.exists?(full_path) | |
FileUtils.mkdir_p File.dirname(full_path) | |
FileUtils.mv temp_file.path, full_path, :force => true | |
rescue Exception => e | |
File.unlink full_path if File.exists?(full_path) | |
output "ERROR: #{e.message}" | |
raise "Failed to download file" | |
end | |
end | |
end # Downloader | |
if __FILE__ == $0 | |
usage = <<-EOU | |
usage: ruby download.rb URL FILE | |
URL http/https/ftp location of the file to download | |
FILE full local path at which to save downloaded file | |
influential environment variables: | |
HTTP_PROXY url to http proxy | |
CA_CERT_FILE full path to CA certificate file | |
EOU | |
abort usage if ARGV.length != 2 | |
Downloader.download_file(ARGV[0], ARGV[1]) | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment