Django SQLite database backend storing the db file on s3.

s3sqlite.py

# -*- coding: utf-8 -*-
import logging
import os
from tempfile import gettempdir

import boto3
import botocore
from django.db.backends.sqlite3.base import DatabaseWrapper

log = logging.getLogger(__name__)


def abspath_join(*args):
    return os.path.abspath(os.path.join(*args))


class DatabaseWrapper(DatabaseWrapper):
    """
    Django SQLite database backend storing the db file on s3.

    Inspired by:
    https://blog.zappa.io/posts/s3sqlite-a-serverless-relational-database

    DATABASES = {
        'default': {
            'ENGINE': '...s3sqlite',
            'NAME': 's3-bucket-name/sqlite.db',
        }
    }
    """
    def __init__(self, *args, **kwargs):
        super(DatabaseWrapper, self).__init__(*args, **kwargs)
        # create lazy handle for remote object
        self.s3obj = self.get_s3_object()
        # set name of local copy
        self.local_name = abspath_join(gettempdir(), self.s3obj.key)

    def get_s3_resource(self):
        # get s3 client
        signature_version = self.settings_dict.get("SIGNATURE_VERSION", "s3v4")
        return boto3.resource('s3', config=botocore.client.Config(
            signature_version=signature_version
        ))

    def get_s3_object(self):
        # get s3 object handle
        bucket_name, key = self.settings_dict['NAME'].split('/', maxsplit=1)
        return self.get_s3_resource().Object(bucket_name, key)

    def get_new_connection(self, conn_params):
        log.info('get_new_connection(conn_params=%s)', conn_params)
        # download if necessary
        self.download_database()
        # point the default sqlite client/db wrapper to our local copy
        conn_params['database'] = self.local_name
        return super(DatabaseWrapper, self).get_new_connection(conn_params)

    def download_database(self):
        # download database to temp file if necessary
        if not os.path.isfile(self.local_name):
            log.debug('downloading database')
            self.s3obj.download_file(self.local_name)

    def close(self, *args, **kwargs):
        log.info('close()')
        super(DatabaseWrapper, self).close()
        self.upload_database()

    def upload_database(self):
        # upload the local copy on every close
        if os.path.isfile(self.local_name):
            log.debug('uploading database')
            self.s3obj.upload_file(self.local_name)