Command Line Backups Solution

backup.py

"""Backup utility

Performs a backup of files and directories given in a YAML configuration file. This is done by first putting them all in a tar archive
and then using the smbclient package to copy the file to the destination.

Run with: python3 backup.py -c /path/to/config.yaml

requires jinja2 and pyyaml """
import argparse
import os
import os.path
import subprocess
import sys
from datetime import datetime

# try the 3rd party imports
try:
    import jinja2
    import yaml
except ImportError:
    print("Install dependencies with 'pip3 install jinja2 pyyaml'")
    sys.exit(1)

# paths
DIR_PATH = os.path.dirname(os.path.realpath(__file__))
TMP_DIR = '/tmp'

# global vars
jinja = jinja2.Environment()

class Command:
    """A system command as defined in the YAML config"""
    _command = None

    def __init__(self, args):
        """stores the full system command as an array of strings"""
        self._command = args

    def __render_template(self, t_string, jinja_vars):
        template = jinja.from_string(t_string)

        return template.render(jinja_vars)

    def generate_command(self, jinja_vars):
        """generate the system command by rendering variables with jinja"""
        result = []

        # render each string in the command
        for s in self._command:
            result.append(self.__render_template(s, jinja_vars))

        return result


def run_process(command):
    """
    Kicks off a subprocess to run the defined program
    with the given arguments. Returns subprocess output.
    """
    # print(command)
    # run process, pipe all output
    output = subprocess.run(command, encoding="utf-8", stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    return output

def run_command_hook(commands, global_config, steps):
    """go through the steps and run each successive command"""
    for step in steps:
        # get the command
        if(step['command'] in commands):
            command_obj = commands[step['command']]

            o = run_process(command_obj.generate_command({"config": global_config,
                                                            "args": step['args']}))
            if(o.returncode != 0):
                print(f"Error executing command {step['command']}")
                print(f"Exiting with error {o.stderr}")
                sys.exit(2)
        else:
            print(f"Command {step['command']} is not defined, aborting")
            sys.exit(2)

def custom_yaml_loader(loader, node):
    """loads another yaml file in the same directory as this one using !include syntax"""
    yaml_file = loader.construct_scalar(node)
    return read_yaml(os.path.join(DIR_PATH, yaml_file))

def check_exists(p, is_dir=False):
    """check that this path exists and either is or isn't a directory"""
    if(os.path.exists(p) and os.path.isdir(p) == is_dir):
        return p
    else:
        raise argparse.ArgumentTypeError(f"Path '{p}' does not exist")


def check_file_exists(f):
    return check_exists(f)


def check_dir_exists(d):
    return check_exists(d, True)

def read_yaml(file):
    result = {}

    try:
        with open(file, 'r') as f:
            result = yaml.safe_load(f)
    except Exception:
        print(f"Error parsing YAML file {file}")

    return result

def write_file(file, pos):
    try:
        with open(file, 'w') as f:
            f.write(str(pos))
    except Exception:
        logging.error('error writing file')


def main():

    # parse the command line arguments
    parser = argparse.ArgumentParser(description='Config File Backup')
    parser.add_argument('-c', '--config', required=True, type=check_file_exists, help='path to config file')
    parser.add_argument('-v', '--verify', action='store_true', help='if a verification file should be left in the calling directory')
    args = parser.parse_args()

    print("Backup")
    print(DIR_PATH)
    print(f"Config file: {args.config}")

    # add custom processor for external loading
    yaml.add_constructor('!include', custom_yaml_loader, Loader=yaml.SafeLoader)
    config_file = read_yaml(args.config)

    # create some variables
    archive_name = f"{config_file['archive_name']}.tar.gz"
    commands = {}
    global_config = {}

    # load any global config variables
    if('config' in config_file):
        global_config = config_file['config']

    # create the commands
    if('commands' in config_file):
        for c in config_file['commands']:
            print(f"Loaded command {c}")
            commands[c] = Command(config_file['commands'][c])

    # check if there is a pre-backup process to run
    if('pre_backup' in config_file):
        print("Running pre-backup")
        run_command_hook(commands, global_config, config_file['pre_backup'])

    # create a tar archive of all the files
    print("Running backup")
    run_process(['tar', 'czf', os.path.join(TMP_DIR, archive_name)] + config_file['files'])

    # check if there is a post-backup process to run
    if('post_backup' in config_file):
        print("Running post-backup")
        run_command_hook(commands, global_config, config_file['post_backup'])

    # move the file to it's destination
    if(config_file['destination'] in commands):
        print("Copying archive")
        copy_command = commands[config_file['destination']]

        # run the file copy command
        o = run_process(copy_command.generate_command({"config": global_config,
                                                        "args": {"source": os.path.join(TMP_DIR, archive_name),
                                                                 "destination": archive_name}}))
        if(o.returncode != 0):
            print("Error copying archive file")
            print(f"{o.stderr}")
            sys.exit(2)
    else:
        print("Command to copy backup archive does not exist")
        sys.exit(2)

    # remove the tmp file
    os.remove(os.path.join(TMP_DIR, archive_name))
    
    if(args.verify):
        write_file(f"{os.path.join(DIR_PATH, config_file['archive_name'])}.complete", f"Complete: { datetime.now() }")

if __name__ == '__main__':
    main()

backup_commands.yaml

mysql_dump:
  - "mysqldump"
  - "-u"
  - "{{ config.mysql_username }}"
  - "-p{{ config.mysql_password }}"
  - "{{ args.database }}"
  - "--result-file={{ args.path }}/{{ args.database }}.sql"
smb_copy:
  - "smbclient"
  - "{{ config.smb_share }}"
  - "{{ config.smb_password }}"
  - "-U"
  - "{{ config.smb_username }}"
  - "-c"
  - "put {{ args.source }} {{config.smb_path}}/{{ args.destination }}"

server_config.yaml

archive_name: servername
config:
  smb_share: //path/to/share
  smb_username: username
  smb_password: pass
  smb_path: /path/to/archive
  mysql_username: username
  mysql_password: pass
commands: !include backup_commands.yaml
destination: smb_copy
pre_backup:
  # do a database dump to a local directory
  - command: mysql_dump
    args:
      database: db_name
      path: /home/user/DB_Backups/
files:
  # backup the database files
  - /home/user/DB_Backups/
  - /path/to/other/files