Backup of lin-group CHPC resources to remote location

chpc_rclone_backup.py

#!/usr/bin/env python3
# Automated backup of lin-group CHPC resources
# Ben Fasoli
#
# Requires rclone to be installed
# Depends on access_token found in ~/.config/rclone/rclone.conf
# To create an access token, run
#   module load rclone
#   rclone config
#     Name: {anything short, e.g. gcloud}
#     Storage: 9 {Google Drive}
#     client_id: {blank}
#     client_secret: {blank}
#     Auto config: n
#     Verification code: {log into provided url, copy key here}
#     Configure as team: n

import os


def find_rclone():
    """
    Identify rclone binary since $PATH may be unreliable for cron jobs
    """
    installdir = '/uufs/chpc.utah.edu/sys/installdir/rclone/'
    versions = []
    for x in os.listdir(installdir):
        if '.' in x and os.path.isdir(os.path.join(installdir, x)):
            versions = versions + [x]
    versions.sort()
    return os.path.join(installdir, versions[-1], 'bin', 'rclone')


def find_rclone_conf():
    """
    Identify .rclone.conf file containing setup information
    """
    conf = os.path.join('~', '.config', 'rclone', 'rclone.conf')
    conf = os.path.expanduser(conf)
    try:
        open(conf)
    except IOError as e:
        print(os.strerror(e.errno) + ': ' + conf)
    return conf


def rclone_copy(local, remote, rclone=find_rclone(), conf=find_rclone_conf()):
    """
    Sync the local directory to the remote location
    """
    # Determine name of remote resource from first line of config
    with open(conf) as f:
        resource = f.readline()
    # Remove brackets and hidden characters from resource
    for string in ['[', ']', '\n', '\r']:
        resource = resource.replace(string, '')
    template = ('{rclone} copy '
                '--config={conf} '
                '{local} {resource}:{remote}'
                '--transfers=2 '
                '--checkers=2 '
                '--bwlimit=5M '
                '--tpslimit 2')
    cmd = template.format(rclone=rclone, conf=conf, local=local, 
                          resource=resource, remote=remote)
    print(cmd)
    os.system(cmd)


if __name__ == '__main__':
    BACKUPS = [
        {
            'name': 'measurements',
            'path': '/uufs/chpc.utah.edu/common/home/lin-group2/measurements'
        },
        {
            'name': 'hrrr',
            'path': '/uufs/chpc.utah.edu/common/home/lin-group6/hrrr/data'
        }
    ]
    REMOTE_DIRECTORY = 'CHPC_Backups'
    for x in BACKUPS:
        remote = os.path.join(REMOTE_DIRECTORY, x['name'])
        rclone_copy(x['path'], remote)