Safe copies a fileset with retry (uses oii utils)

safe_copy_fileset.py

import random
import shutil
import tempfile
import os

from oii.utils import retry, safe_tempdir, gen_id, compare_files

def unreliable_copy(src,dest):
    if random.random() > 0.2:
        shutil.copy(src,dest)
    else:
        with open(dest,'w') as fout:
            fout.write('bogus')

def safe_copy_fileset(srcdests):
    @retry(IOError)
    def reliable_copy(src,dest):
        unreliable_copy(src, dest) # fixme shutil.copy
        if not compare_files(src, dest, name=False, size=True):
            raise IOError('copying %s to %s failed' % (src,dest))
    with safe_tempdir() as tempdir:
        src_dest_tmp = [(src, dest, os.path.join(tempdir, gen_id())) for src, dest in srcdests]
        for src, _, tmp in src_dest_tmp:
            reliable_copy(src,tmp)
        for _, dest, tmp in src_dest_tmp:
            dest_dir = os.path.dirname(dest)
            try:
                os.makedirs(dest_dir)
            except:
                pass
            if not os.path.isdir(dest_dir):
                raise IOError('unable to create directory %s' % dest_dir)
            shutil.move(tmp,dest)