Skip to content

Instantly share code, notes, and snippets.

@nkrabben
Created May 9, 2019 14:09
Show Gist options
  • Save nkrabben/ad9336346e82436c8c743d4bbe8eac02 to your computer and use it in GitHub Desktop.
Save nkrabben/ad9336346e82436c8c743d4bbe8eac02 to your computer and use it in GitHub Desktop.
Multiprocess thinger that uses a find exec command to avoid too many argument errors
import os
import subprocess
import multiprocessing
import timeit
def work(drive):
cmd = [
'find',
drive,
'-path',
'*/{E,S}{d,e}*',
'-exec'
'snowball',
'cp',
'{}',
's3://yadayadayada',
';']
try:
subprocess.check_call(cmd)
except:
print("Didn't work for: {}", drive)
return
def main():
tic = timeit.default_timer()
drives = [os.path.join('/Volumes', x) for x in os.listdir('/Volumes') if x != 'Macintosh HD']
print(drives)
pool = multiprocessing.Pool(processes=len(drives))
r = pool.map_async(work, drives)
r.wait()
toc = timeit.default_timer()
print("It took this long: {}".format(tic-toc))
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment