This Python script monitors a given folder (recursive) for changes and synchronizes them with a remote station. Additionally to that it also (re-)starts a tmux session with the given main application script after a change.

distribute_files.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
A tool to distribute currently developed files to a remote station, 
whenever they are changed.
 
Usage:
    distribute_files.py PATH APP [options]
    distribute_files.py -c CONFIG [options] 
    distribute_files.py [PATH] [APP] -c CONFIG [options] 
    distribute_files.py PATH APP [options] --config FILE
 
Arguments:
    PATH                      Path to folder to watch after
    APP                       Main application

Options:
    -e FILEENDING             file endings to exclude
    -i FILEENDING             file endings to include only
    -r PATH_TO_REMOTE         path to remote station (e.g. user@stationname:~/path)
    -t SESSION                tmux session name on remote machine
 
Other options:
    -l, --logfile LOGFILE     log file for logging
    -c CONFIG                 loads config file
    --config FILE             creates empty config file
    -q, --quiet               print less text
    -v, --verbose             print more text
    -h, --help                show this help message and exit
    --version                 show version and exit
"""

__author__ = "Jasper Buesch"
__copyright__ = "Copyright (c) 2014"
__version__ = "0.0.3"
__email__ = "[email protected]"

import os
import time
import logging
import json
import sys
import subprocess


def get_file_tree_list(path, exclude_endings=[], include_endings=[]):
    file_list = []
    try:
        for folder, dirs, files in os.walk(path):
            for f in files:
                if ((len(include_endings) == 0) and not f.split(".")[-1] in exclude_endings) or f.split(".")[-1] in include_endings:
                    file_name = folder + "/" + f
                    file_list.append(file_name)
    except OSError:
        return None
    logging.getLogger("get_file_tree_list").debug(file_list)
    return file_list


def get_file_ages(file_list):
    dic = {}
    for f in file_list:
        try:
            dic[f] = os.path.getmtime(f)
        except OSError:
            print "Error, file " + f + " doesn't exist anymore"
    logging.getLogger("get_file_ages").debug(dic.keys())
    return dic


def check_for_changes(watch_path, file_ages, exclude_endings=[], include_endings=[]):
    log = logging.getLogger('check_for_changes')
    current_file_times = get_file_ages(get_file_tree_list(watch_path, exclude_endings=exclude_endings, include_endings=include_endings))
    for f in file_ages.keys():  # delete files that are no longer there
        if f not in current_file_times.keys():
            del file_ages[f]
            log.info("<%s> is no longer there" % (f))
    for f in current_file_times.keys():  # delete files that are no longer there
        if f not in file_ages.keys():
            file_ages[f] = None
            log.info("<%s> newly appeared and is now being watched" % (f))
    new_times = get_file_ages(get_file_tree_list(watch_path, exclude_endings=exclude_endings, include_endings=include_endings))

    for f in file_ages.keys():
        if file_ages[f] is None:
            file_ages[f] = new_times[f]
        elif new_times[f] != file_ages[f]:
            log.info("<%s> has changed" % (f))
            file_ages[f] = new_times[f]
            return f


def synchronize_file(watch_path, file_path, remote_path):
    rel_path = os.path.dirname(os.path.relpath(file_path, watch_path))
    remote_path = remote_path + "/" + rel_path
    if subprocess.check_call(["scp", file_path, remote_path]) == 0:
        logging.getLogger("synchronize_file").info("Successfully synchronized %s to %s" % (file_path, remote_path))
        return True
    return False


def remote_command_execution(remote_machine, command):
    try:
        if subprocess.check_call(["ssh", remote_machine, command]) == 0:
            log = logging.getLogger('remote_command_execution').debug("Successfully called remote ssh command")
            return True
        return False
    except:
        return False


def restart_app_remote(remote_path, app_name, session_name="remote_devel"):
    log = logging.getLogger('restart_app_remote')
    remote_machine = remote_path.split(":")[0]
    if not remote_command_execution(remote_machine, "tmux kill-session -t %s" % (session_name)):
        log.warn("Killing tmux session failed (maybe because there wasn't one before - which'd be fine)")
    if not remote_command_execution(remote_machine, "tmux new-session -d -s %s" % (session_name)) or not remote_command_execution(remote_machine, "tmux send-keys -t %s '%s' C-m" % (session_name, app_name)):
        log.error("Opening a new remote session failed!")
        

def main(args):
    log = logging.getLogger('main')
    log.info("===\nProgram started")
    path = args['PATH']
    app_name = args['APP']
    session_name = args['-t']
    exclude_endings = [args['-e']] if args['-e'] else []
    include_endings = [args['-i']] if args['-i'] else []
    demo_mode = False

    if not '-r' in args:
        remote_path = None
        log.warn("remote machine path is missing (option '-r'), so we run in demo mode")
        demo_mode = True
    else:
        remote_path = args['-r']
    file_times = get_file_ages(get_file_tree_list(path, exclude_endings=exclude_endings, include_endings=include_endings))

    while True:
        time.sleep(1)
        file_changed = check_for_changes(path, file_times, exclude_endings=exclude_endings, include_endings=include_endings)
        if not file_changed is None:
            if demo_mode:
                log.info("Demo mode: We simulize to synchronize file %s" % (file_changed))
            elif not synchronize_file(path, file_changed, remote_path):
                log.info("Synchronization of %s to %s failed" % (file_changed, remote_path))
                continue
            restart_app_remote(remote_path, app_name, session_name=session_name)

# --------------------------------------------------------------------

def parse_json_config(args):
    """
    Takes care of the correct configure file management.
 
    It either prints the empty json config structure or adds the
    parameters from the given one to the existing arguments (args)
    """
    if args['--config']:
        file_name = args['--config']
        del args['--config']
        with open(file_name, 'w') as file_name:
            json.dump(args, file_name, sort_keys=True, indent=4 * len(' '))
        sys.exit()
    if args['-c']:
        json_config = json.loads(open(args['-c']).read())
        return dict((str(key), args.get(key) or json_config.get(key))
            for key in set(json_config) | set(args))
    return args

# --------------------------------------------------------------------

if __name__ == "__main__":
    try:
        from docopt import docopt
    except:
        print """
        Please install docopt using:
            pip install docopt==0.6.1
        For more refer to:
        https://github.com/docopt/docopt
        """
        raise
 
    args = docopt(__doc__, version=__version__)
    args = parse_json_config(args)
 
    FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    log_level = logging.INFO  # default
    if args['--verbose']:
        log_level = logging.DEBUG
    elif args['--quiet']:
        log_level = logging.ERROR
    logging.basicConfig(level=log_level, format=FORMAT)

    log = logging.getLogger('file_distributer')
    log.debug("Arguments:")
    log.debug(args)

    if args['--logfile']:
        filename = args['--logfile']
        days_to_log = 7
        formatter = logging.Formatter(FORMAT)
        time_logger = logging.handlers.TimedRotatingFileHandler(filename, when='midnight', backupCount=days_to_log)
        time_logger.setFormatter(formatter)
        log.addHandler(time_logger)
    main(args)