Quick attempt at incremental transfer of individual files (that only grow in size)

incremental-xfer.py

#!/usr/bin/env python

import os, sys, json
import paramiko

class Transfer:
    def __init__(self, srcfile, dsthost, dstdir):
        self.src_file = srcfile
        self.dst_host = dsthost
        self.dst_dir = dstdir
        self._index = None

        self._ssh = paramiko.SSHClient()
        self._ssh.set_missing_host_key_policy(paramiko.WarningPolicy())
        self._ssh.load_system_host_keys()
        self.find_offset()

    def connect(self):
        agent = paramiko.Agent()
        for key in agent.get_keys():
            try:
                self._ssh.connect(self.dst_host, pkey=key, compress=True)
                return self._ssh.open_sftp()
            except paramiko.SSHException:
                raise Exception("Connection to %s failed!" % (dsthost))
    
        raise Exception("An unknown error occurred")
    
    def find_offset(self):
        dirname = os.path.dirname(self.src_file)
        filename = os.path.basename(self.src_file)
        indexfile = os.path.join(dirname, '.xferindex')
        new_index = False

        if os.path.exists(indexfile):
            fd = open(indexfile, 'a+')
            try:
                self._index = json.loads(fd.read())
            except:
                self._index = {}
        else:
            fd = open(indexfile, 'a+')
            self._index = {}
        
        if filename not in self._index:
            self._index[filename] = 0
            fd.seek(0)
            fd.write(json.dumps(self._index))

        fd.close()
        return self._index[filename]
    
    def copy(self):
        offset = self.find_offset()
        filename = os.path.join(self.dst_dir, os.path.basename(self.src_file))

        lfd = open(self.src_file, 'r')
        lfd.seek(offset)
        
        sftp = self.connect()
        rfd = sftp.open(filename, 'a+')
        rfd.seek(offset)
        rfd.write(lfd.read())
        
        lfd.close()
        rfd.close()

def main(argv=None):
    if argv is None:
        argv = sys.argv
    
    if len(argv) != 4:
        print "%s <src file> <dst host> <dst dir>" % (argv[0])
        return -1
    
    xfer = Transfer(argv[1], argv[2], argv[3])
    xfer.copy()
    
    return 0

if __name__ == '__main__':
    sys.exit(main())