split transfer big files (for some poor disks)

checkmd5.py

"""
    check md5 of every part, and remove broken part
    put in same foler with partitions, and mkdir part, with empty file md5ok.txt and hash file md5sum.txt
"""

import os

files = os.listdir()
files.sort()
files = files[:-4]
#print(files)
truemd5l = [x.split(' ') for x in open('md5sum.txt').read().strip().split('\n')]
#print(truemd5l)
truemd5 = {}
for i, _, j in truemd5l:
    truemd5[j] = i
#print(truemd5)

md5ok = open('md5ok.txt').read().strip().split(' ')

for file in files:
    if file in md5ok:
        del truemd5[file]
        continue
    res = os.popen('md5sum ' + file).read().split(' ')[0]
    #print(file, res, truemd5[file])
    if res == truemd5[file]:
        res = 'Passed'
        md5ok.append(file)
    else:
        res = 'Failed'
        os.system('rm -f ' + file)
    print(file, res)
    del truemd5[file]

k = list(truemd5.keys())
k.sort()
for file in k:
    print(file, end=' ')
print('not Exist')

open('md5ok.txt', 'w').write(' '.join(md5ok))

combine.py

"""
    put in part/, will combine every part's part to parent folder of part/
"""

import os

expected_part = 41

files = os.listdir()
files.sort()

m = {}
for i in files[:-1]:
    m[i[:-7]] = []
for i in files[:-1]:
    m[i[:-7]].append(i)

#print(m)

for key in m:
    if len(m[key]) == expected_part:
        if (not os.path.exists('../' + key)):
            print('combine', key)
            os.system('cat %s.* > ../%s' % (key, key))
            print(key, 'combined')

script.sh

#put in part/

source ~/activate

python combine.py

cd ..
python check.py

for i in Clickture.*; do rm part/$i* 2>/dev/null && echo $i part removed; done

cd part

transfer.py

"""
    transfer $1 to $2/$1.%06d, splitting $1 into small parts
"""

import os
import sys
import random
import time

os.system('~/remount.sh')

filename = sys.argv[1]
fsize = os.path.getsize(filename)

blocksize = 100 * 1024 * 1024
nowpos = 0
count = 0

if len(sys.argv) > 3:
    count = int(sys.argv[3])
    nowpos = count * blocksize

def randstr(length = 32):
    base = "0123456789qazwsxedcrfvtgbyhnujmikolp"
    s = '/tmp/'
    for i in range(length):
        s += base[random.randint(0, len(base) - 1)]
    return s

'''
file = open(filename, 'rb')
folder = randstr()
os.mkdir(folder)

while nowpos < fsize:
    pname = '%s/%s.%06d' % (folder, filename, count)
    file.seek(nowpos, 0)
    with open(pname, 'wb') as f:
        f.write(file.read(blocksize))
    count += 1
    nowpos += blocksize
'''

target = sys.argv[2]
tmpfile = randstr()

fails = []

while nowpos < fsize:
    pname = '%s/%s.%06d' % (target, filename.split('/')[-1], count)
    wait_expand = 1
    starttime = 0
    while True:
        try:
            starttime = time.time()
            with open(filename, 'rb') as file:
                file.seek(nowpos, 0)
                with open(tmpfile, 'wb') as f:
                    f.write(file.read(blocksize))
            wait_expand = 1
            break
        except:
            if wait_expand > 5:
                fname = filename.split('/')[-1] + '.' + '%06d' % (count)
                print('Fail too many times! skip it.', fname)
                fails.append(fname)
                break
            print('open file error! wait %ds and remount' % (30 * wait_expand))
            time.sleep(30 * wait_expand)
            wait_expand += 1
            os.system('~/remount.sh')
    if os.path.getsize(tmpfile) != 0:
        cmd = 'rsync -P --rsh=ssh %s %s' % (tmpfile, pname)
        print(pname, time.time() - starttime)
        os.system(cmd)
    count += 1
    nowpos += blocksize
    time.sleep(2)

os.remove(tmpfile)

if len(fails) > 0:
    fails.append('')

with open('/tmp/transferfail.log', 'a') as f:
    f.write('\n'.join(fails))