gistfile1.txt

class S3PickleAndLocalCache(object):
    
    def __init__(self, local_on=True):
        self.bucket_name = settings.S3_CACHE_BUCKET
        self.conn = boto.s3.connection.S3Connection(
            settings.AWS_ACCESS_KEY_ID,
            settings.AWS_SECRET_ACCESS_KEY
        )
        try:
            self.bucket = self.conn.get_bucket(self.bucket_name, validate=True)
        except boto.exception.S3ResponseError:
            self.bucket = self.conn.create_bucket(self.bucket_name)
            
        self.local_on = local_on
        if self.local_on:
            self.local_cache = cache.get_cache('filebased')
    
    def clean_key(self, key):
        return key.replace('/','-').replace(' ', '-')
    
    def get(self, key):
        # try local first because network calls are expensive
        if self.local_on:
            k = self.local_cache.get(self.clean_key(key))
            if k is not None:
                return k
        
        # try s3 to see if another machine has 
        k = self.bucket.get_key(key)
        
        if k is None:
            return None
        
        try:
            val = pickle.loads(k.get_contents_as_string())
        except Exception as e:
            print e
            print '-----'
            print '-----'
            print '-----'
            print '-----'
            print 'cache error, retrying'
            print '-----'
            print '-----'
            print '-----'
            print '-----'
            sleep(5)
            try:
                k = self.bucket.get_key(key)
                val = pickle.loads(k.get_contents_as_string())
                print 'success on retry'
            except Exception as e:
                print 'failed retry'
                return None
        # set to local to avoid further network calls
        if self.local_on:
            self.local_cache.set(self.clean_key(key), val)
        return val