Created
December 2, 2015 19:28
-
-
Save halloffame/f36795a44fe9d160f2bc to your computer and use it in GitHub Desktop.
Ruby script to recursively download a given directory from your dropbox.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require 'dropbox_sdk' # gem install dropbox-sdk | |
require 'fileutils' | |
require 'thread' | |
# Example: downloads entire dropbox | |
# DropboxDump.new('your_access_token', '/download/destination').download("/") | |
class DropboxDump | |
attr_accessor :client, :dest, :work_q, :worker_count | |
def initialize(access_token, dest, max_workers = 3) | |
@client = DropboxClient.new(access_token) | |
@dest = dest | |
@work_q = Queue.new | |
# Dropbox has a request limit on their API so if you do too many workers it might go over the limit and start failing. | |
@worker_count = max_workers | |
end | |
def download(path) | |
download_directory(path) | |
workers = (0...worker_count).map do | |
Thread.new do | |
begin | |
while file = work_q.pop(true) | |
if file | |
download_file(file) | |
end | |
end | |
rescue ThreadError | |
# if queue is empty, it raises an error, but we don't really care | |
rescue => e | |
puts "Error: #{e}" | |
end | |
end | |
end | |
workers.map(&:join); "ok" | |
end | |
def download_directory(path) | |
# Dropbox isn't case sensitive, so we are just downcasing all the file paths so we don't run into problems. | |
path = path.downcase | |
folder_path = "#{dest}#{path}" | |
# Create the full directory path if it is not already there | |
FileUtils.mkdir_p(folder_path) | |
puts folder_path | |
# Loop through each file for the given path | |
files = client.metadata(path)['contents'] | |
files.each do |file| | |
if file["is_dir"] | |
download_directory(file['path'].downcase) | |
else | |
work_q.push(file) | |
end | |
end | |
end | |
def download_file(file) | |
# See comment above about downcasing | |
file_path = file['path'].downcase | |
local_path = "#{dest}#{file_path}" | |
if file["is_dir"] | |
download_directory(file_path) | |
else | |
# download file | |
if File.file?(local_path) | |
puts " -> File Exists: #{local_path}" | |
else | |
begin | |
contents, metadata = client.get_file_and_metadata(file_path) | |
open(local_path, 'w') {|f| f.puts contents } | |
puts " -> Saved: #{local_path}" | |
rescue => e | |
puts "ERROR saving #{local_path}" | |
puts "metadata: #{file}" | |
puts "error:" | |
puts e | |
end | |
end | |
end | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment