Simple, unsubtle hack to back up files matching two patterns to AWS. Designed for a Django project right now, but no real need for the coupling.

AWS Backup

#!/usr/bin/env python

import datetime
import os
import time
from zipfile import ZipFile, ZIP_DEFLATED

from django.conf import settings
from django.core.mail import mail_admins
from django.core.management.base import BaseCommand

from boto.s3.connection import S3Connection
from boto.s3.key import Key

DAYS_TO_KEEP_FILES = 5
DEBUGGING = False

def display_progress(bytes_sent, total_bytes):
    print "%.1f%% complete" % (100.0 * (bytes_sent / total_bytes))


class Command(BaseCommand):
    """
    Back up database to S3 Storage. Boto docs at:
    http://docs.pythonboto.org/en/latest/s3_tut.html
    Also delete files older than 5 days if they aren't the most
    recent backup
    """
    args = "<dry_run>"
    help = """Move most recent database files to Amazon and delete any older files
Dry run = just show files"""

    def handle(self, *args, **options):
        errors = []
        dry_run = len(args) > 0 and args[0] == "yes"
        # create our connection
        conn = S3Connection(settings.AWS_ID, settings.AWS_KEY)
        bucket = conn.get_bucket(settings.AWS_BUCKET)
        threshold = time.mktime(
            (datetime.datetime.now() - datetime.timedelta(days=DAYS_TO_KEEP_FILES)
            ).timetuple())

        directory = settings.DATABASE_BACKUP_PATH
        files = [os.path.join(directory, f) for f in os.listdir(directory) if not os.path.isdir(f)]
        to_backup = {}
        if files:
            # find most recent files matching
            for name, pattern in settings.AWS_BACKUP_PATTERNS.iteritems():
                try:
                    to_backup[name] = max([f for f in files if f.find(pattern) > -1], key=lambda x: os.stat(x).st_mtime)
                except ValueError:
                    errors.append("Could not find a %s file (pattern = '%s')" % (name, pattern))
            to_delete = [f for f in files if f not in to_backup and os.path.getmtime(f) < threshold]
        else:
            errors.append("NO FILES FOUND!")
        if dry_run:
            if errors:
                print "ERRORS ENCOUNTERED - would not have completed/ deleted any files"
                for e in errors:
                    print e
            print "Would have backed up the following:"
            for name, pattern in settings.AWS_BACKUP_PATTERNS.iteritems():
                if name in to_backup:
                    f = to_backup[name]
                    print "%s: %s (%s)" % (name, f, time.ctime(os.stat(f).st_mtime))
            print "TO DELETE:"
            for d in to_delete:
                print d
            return
        for name, filepath in to_backup.iteritems():
            if not self._backup_file(bucket, filepath, name):
                errors.append("Did not backup %s" % filepath)
        if errors:
            body = "The following errors were encountered during today's backup:" + os.linesep
            for e in errors:
                body += e + os.linesep
            body += "No files were deleted because of the errors"
            print body
            if not DEBUGGING:
                mail_admins("Amazon Database Backup Problems", body, fail_silently=True)
            return
        if not DEBUGGING:
            for d in to_delete:
                try:
                    os.unlink(d)
                except Exception:
                    pass
    
    def _backup_file(self, bucket, filepath, key):
        if not filepath:
            return False
        print "Starting backup of %s" % filepath
        k = Key(bucket)
        k.key = key
        zip_name = filepath + ".zip"
        try:
            with ZipFile(zip_name, "w") as myzip:
                myzip.write(filepath, os.path.basename(filepath), ZIP_DEFLATED)
            k.set_contents_from_filename(zip_name, cb=display_progress)
            try:
                os.unlink(zip_name)
            except Exception:
                pass
            return True
        except Exception:
            pass

The callback function is either useless or I'm getting round off problems:

0.0% complete
0.0% complete
0.0% complete
0.0% complete
0.0% complete
0.0% complete
0.0% complete
0.0% complete
0.0% complete
100.0% complete