Django management command to tar files and send to S3

backup_data_to_amazon.py

import datetime
import os
import tarfile
import time

from django.conf import settings
from django.core.mail import mail_admins
from django.core.management.base import BaseCommand

from boto.s3.connection import S3Connection
from boto.s3.key import Key

DAYS_TO_KEEP_FILES = 2
DEBUGGING = False


def display_progress(bytes_sent, total_bytes):
    print "%.1f%% complete" % (100.0 * (bytes_sent / total_bytes))


class Command(BaseCommand):
    """
    Back up database to S3 Storage. Boto docs at:
    http://docs.pythonboto.org/en/latest/s3_tut.html
    Also delete files older than DAYS_TO_KEEP_FILES days if they aren't the most recent backup
    """
    args = "<dry_run>"
    help = """Move most recent database files to Amazon and delete any older files
Dry run = just show files"""

    def handle(self, *args, **options):
        errors = []
        dry_run = len(args) > 0 and args[0] == "yes"
        # create our connection
        conn = S3Connection(settings.AWS_ID, settings.AWS_KEY)
        bucket = conn.get_bucket(settings.AWS_BUCKET)
        threshold = time.mktime(
            (datetime.datetime.now() - datetime.timedelta(days=DAYS_TO_KEEP_FILES)
            ).timetuple())

        directory = settings.DATABASE_BACKUP_PATH
        files = [os.path.join(directory, f) for f in os.listdir(directory) if not os.path.isdir(f)]
        to_backup = {}
        to_delete = []
        if files:
            # find most recent files matching
            for name, pattern in settings.AWS_BACKUP_PATTERNS.iteritems():
                try:
                    to_backup[name] = max([
                            f for f in files if f.find(pattern) > -1 and f.find(".tar.") == -1
                        ], key=lambda x: os.stat(x).st_mtime)
                except ValueError:
                    errors.append("Could not find a %s file (pattern = '%s')" % (name, pattern))
            to_delete = [f for f in files if f not in to_backup and os.path.getmtime(f) < threshold]
        else:
            errors.append("NO FILES FOUND!")
        if dry_run:
            if errors:
                print "ERRORS ENCOUNTERED - would not have completed/ deleted any files"
                for e in errors:
                    print e
            print "Would have backed up the following:"
            for name, pattern in settings.AWS_BACKUP_PATTERNS.iteritems():
                if name in to_backup:
                    f = to_backup[name]
                    print "%s: %s (%s)" % (name, f, time.ctime(os.stat(f).st_mtime))
            print "TO DELETE:"
            for d in to_delete:
                print d
            return
        for name, filepath in to_backup.iteritems():
            if not self._backup_file(bucket, filepath, name):
                errors.append("Did not backup %s" % filepath)
        if errors:
            body = "The following errors were encountered during today's backup:" + os.linesep
            for e in errors:
                body += e + os.linesep
            body += "No files were deleted because of the errors"
            print body
            if not DEBUGGING:
                mail_admins("Amazon Database Backup Problems", body, fail_silently=True)
            return
        if not DEBUGGING:
            for d in to_delete:
                try:
                    os.unlink(d)
                except Exception:
                    pass

    def _backup_file(self, bucket, filepath, key):
        """
        Back SQL files to Amazon after tar/gzip
        """
        if not filepath:
            return False
        print "Starting backup of %s" % filepath
        k = Key(bucket)
        k.key = key
        tar_name = filepath + ".tar.bz2"
        tar = tarfile.open(tar_name, "w:bz2")
        tar.add(filepath)
        k.set_contents_from_filename(tar_name, cb=display_progress)
        return True