A wrapper for Boto that provides a posix-like management interface to S3 with Read, Write, and Delete.

s3manager.py

#!/usr/bin/env python

# graffiti.breakdance.s3
# Handler class for writing to Amazon S3
#
# Author:   Benjamin Bengfort <[email protected]>
# Created:  Fri Sep 27 08:48:19 2013 -0400
#
# Copyright (C) 2013 Cobrain Company
# For license information, see LICENSE.txt
#
# ID: s3.py [] [email protected] $

"""
A client that performs a RESTful request to the S3 API with Boto to write
data files to a long term data store.

    >>> manager = S3Manager()
    >>> manager.write("test.txt", "Shopping list: Apples, Oranges, Pears")
    >>> manager.read("test.txt")
    'Shopping list: Apples, Oranges, Pears'
    >>> manager.delete("test.txt")
"""

##########################################################################
## Imports
##########################################################################

import boto
import random
import string

from datetime import datetime
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured

##########################################################################
## Exception Heirarchy
##########################################################################

class AWSClientError(Exception):
    """
    There was some problem connecting to AWS
    """
    pass

##########################################################################
## Writer Class
##########################################################################

class S3Manager(object):

    def __init__(self, bucket=None,
                 aws_access_key_id=None, aws_secret_access_key=None,
                 **kwargs):

        self.bucket = bucket
        self.aws_access_key_id = aws_access_key_id
        self.aws_secret_access_key = aws_secret_access_key
        self.connect()

    ##////////////////////////////////////////////////////////////////////
    ## Instance properties
    ##////////////////////////////////////////////////////////////////////

    @property
    def bucket(self):
        """
        Note: could return either a string or a boto.Bucket object. Always
        perform type checking or duck checking (try/except) when using this
        property.
        """
        return self._bucket

    @bucket.setter
    def bucket(self, bucket):
        """
        Setter for the bucket object, if connected to AWS, this creates a
        boto.Bucket object and sets that, otherwise simply saves the bucket
        name as a string for use during connection.
        """
        bucket = bucket or getattr(settings, "S3_BUCKET", "")
        if not bucket:
            raise ImproperlyConfigured("No S3_BUCKET has been specified.")

        if self.is_connected():
            self._bucket = self._connection.lookup(bucket)
            if not self._bucket:
                self.close()
                raise ImproperlyConfigured("Could not connect to S3 Bucket "
                                           " named %s" % bucket)
            self._test_write()
        else:
            self._bucket = bucket

    @property
    def aws_access_key_id(self):
        return self._aws_access_key_id

    @aws_access_key_id.setter
    def aws_access_key_id(self, key):
        """
        TODO: Make a settings fetch descriptor
        """
        key = key or getattr(settings, "AWS_ACCESS_KEY_ID", "")
        if not key:
            raise ImproperlyConfigured("No AWS_ACCESS_KEY_ID has been specified")
        self._aws_access_key_id = key

    @property
    def aws_secret_access_key(self):
        return self._aws_secret_access_key

    @aws_secret_access_key.setter
    def aws_secret_access_key(self, key):
        """
        TODO: make a settings fetch descriptor
        """
        key = key or getattr(settings, "AWS_SECRECT_ACCESS_KEY", "")
        if not key:
            raise ImproperlyConfigured(("No AWS_SECRECT_ACCESS_KEY "
                                        "has been specified"))
        self._aws_secret_access_key = key

    ##////////////////////////////////////////////////////////////////////
    ## API Methods
    ##////////////////////////////////////////////////////////////////////

    def read(self, path):
        """
        Read data from a path - specified in the posix style. If the path
        does not exist in S3 an error will be raised.

        NOTE: When dealing with large files, it is going to be better to
        save the contents to a file using other boto methods.

        :param path: A posix style path to a location in the bucket
        :returns: A string with the data from the path
        """
        key = self.get_s3_key(path)
        if not key: raise AWSClientError("Could not read from S3 bucket at '%s'" % path)
        return key.get_contents_as_string()

    def write(self, path, data, overwrite=True):
        """
        Write data to a path - specified in the posix style. This will
        cause a write to the bucket as defined in the settings or the init
        of the manager.

        NOTE: When dealing with large files, it is going to be better to
        write the content from a file using other boto methods, and ensure
        that the file is compressed.

        :param path: A posix style path to a location in the bucket
        :param data: The data to write to the bucket at the path.
        :param overwrite: Specify whether or not to overwrite existing data
        :returns: None
        """
        if data:
            key = self.get_s3_key(path, overwrite=overwrite)
            if not key: raise AWSClientError("Could not write to S3 bucket at '%s'" % path)
            key.set_contents_from_string(data)

    def delete(self, path):
        """
        Delete a file in S3 at a path - specified in the posix style. If
        the path does not exist in S3 an error will be raised.

        :param path: A posix style path to a location in the bucket
        :returns: None
        """
        key = self.get_s3_key(path)
        if not key: raise AWSClientError("Could not delete '%s' from S3 bucket" % path)
        key.delete()

    def is_connected(self):
        """
        Always ensure that if there is an error, call `self.close`

        :returns: Boolean describing state of connection
        """
        return (hasattr(self, '_connection') and
                getattr(self, '_connection'))

    ##////////////////////////////////////////////////////////////////////
    ## Helper methods
    ##////////////////////////////////////////////////////////////////////

    def connect(self, bucket=None):
        """
        Creates a connection object to AWS and instantiates the Bucket
        object, which in turn tests the writability of the bucket in S3
        """
        try:
            self._connection = boto.connect_s3( self.aws_access_key_id,
                                                self.aws_secret_access_key )
        except boto.exception.BotoClientError as e:
            raise AWSClientError(str(e))

        self.bucket = bucket or self.bucket

    def close(self):
        """
        Close the connection and clean up the class.
        """
        try:
            self._connection.close()
        except:
            pass
        self._connection = None

    def get_s3_key(self, name, overwrite=True):
        """
        Get the key (sort of like the path) in s3. This is an object that
        is like a file handler to a particular point in s3. If you want to
        overwrite (the default), simply use it, if not overwrite, test for
        None.
        """
        key = boto.s3.key.Key(self.bucket, name)
        if not overwrite:
            if key.exists(): return None
        return key

    def _test_write(self):
        """
        Create a randomly named key with the timestamp, then attempt to
        set the contents from a string and delete it. If failure, raise
        AWSClientError.
        """
        def random_chars(size=6):
            chars = string.ascii_letters + string.digits
            return ''.join(random.choice(chars) for x in xrange(size))

        try:
            key = "%s-%s.txt" % (datetime.now().strftime("%Y%m%d%H%M%S"),
                                 random_chars())
            key = self.get_s3_key(key, overwrite=True)
            key.set_contents_from_string(random_chars(32))
            key.delete()
        except Exception as e:
            raise AWSClientError(str(e))