Copy function with progress display in Python

cp_with_prog.py

#!/usr/bin/python

# Copy function with progress display using a callback function.
# The callback function used here mimics the output of
# 'rsync --progress'.

# Copyright (C) 2012 Thomas Hipp

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import datetime
import os

def copy(src, dst, callback=None):
    blksize = 1048576 # 1MiB
    try:
        s = open(src, 'rb')
        d = open(dst, 'wb')
    except (KeyboardInterrupt, Exception) as e:
        if 's' in locals():
            s.close()
        if 'd' in locals():
            d.close()
        raise
    try:
        total = os.stat(src).st_size
        pos = 0
        start_elapsed = datetime.datetime.now()
        start_update = datetime.datetime.now()
        while True:
            buf = s.read(blksize)
            bytes_written = d.write(buf)
            end = datetime.datetime.now()
            pos += bytes_written
            diff = end - start_update
            if callback and diff.total_seconds() >= 0.2:
                callback(pos, total, end - start_elapsed)
                start_update = datetime.datetime.now()
            if bytes_written < len(buf) or bytes_written == 0:
                break
    except (KeyboardInterrupt, Exception) as e:
        s.close()
        d.close()
        raise
    else:
        callback(total, total, end - start_elapsed)
        s.close()
        d.close()

def tmstr(t):
    days, rest = divmod(t, 86400)
    hours, rest = divmod(rest, 3600)
    minutes, seconds = divmod(rest, 60)
    return '{0:4d}:{1:02d}:{2:02d}'.format(int(days * 24 + hours),
            int(minutes), round(seconds))

mod = 101
speed = [0] * mod
index = 0

def progress(pos, total, elapsed):
    global speed
    global index
    global mod
    elapsed_ = elapsed.total_seconds()
    speed[index % mod] = pos / elapsed_
    index = (index + 1) % mod
    out = '{0:12} {1:4.0%} {2:7.2f}{unit}B/s {3}'
    unit = ('Mi', 1048576) if total > 999999 else ('ki', 1024)
    # complete
    if pos == total:
        print(out.format(pos, pos / total, sum(speed) / mod / unit[1],
            tmstr(elapsed_), unit=unit[0]))
    # in progress
    else:
        print(out.format(pos, pos / total, sum(speed) / mod / unit[1],
            tmstr((total - pos) / sum(speed) * mod), unit=unit[0]), end='')
        print('\r')

Thanks for this code, it did not work as is for me neither since I had following issue:

File "./cp_with_prog.py", line 88
tmstr((total - pos) / sum(speed) * mod), unit=unit[0]), end='')
^
SyntaxError: invalid syntax

But after trying with python3 I fixed it changing the header with:
#!/usr/bin/python3
Or calling explicitly:
python3 cp_with_prog.py

Also, I changed blksize to 10485760 (10 MiB) to be less drawn in progress lines outputs and it may need a user input to confirm copy if destination file already exists.

Anyway, this is a good start to have a copy with progress in python.

Edit: after looking around to have only one output line that is updated at each step instead of one per step (getting 100 output lines for a 100 MiB file, which is a lot) I understood the '\r' thing, and I fixed the progress function code adding a first print('', end='\r') line in if pos == total (else the last progress line is displayed in addition to the final one) and setting end='\r' instead of end='' in else print and removing the print('\r') in this same else, the full if/else becoming:

    if pos == total:
        print("", end='\r')
        print(out.format(pos, pos / total, sum(speed) / mod / unit[1],
            tmstr(elapsed_), unit=unit[0]))
    # in progress
    else:
        print(out.format(pos, pos / total, sum(speed) / mod / unit[1],
            tmstr((total - pos) / sum(speed) * mod), unit=unit[0]), end='\r')

Edit 2: I also have seen execution rights are lost upon copy with this system, at least under Linux:

• I added following code at the end of file to copy itself in /tmp:

if __name__ == '__main__':
    copy("/home/user/dev/cp_with_progress.py", "/tmp/cp_with_progress.py", progress)

• And checking source file and target file I can see execution rights are gone on target:

/home/user/dev>ll
-rw**x**r-**x**r-**x** 1 user users       3141 Feb 19 10:39 cp_with_progress.py
/home/user/dev>./cp_with_progress.py
        3024 100%  456.86kiB/s    0:00:00
/home/user/dev>ll /tmp/cp_with_progress.py
-rw**-**r-**-**r-**-** 1 user users 3024 Feb 19 10:58 /tmp/cp_with_progress.py