gistfile1.py

import os.path
import json
import logging

from django.conf import settings
from django.contrib.staticfiles.storage import CachedFilesMixin
from storages.backends.s3boto import S3BotoStorage


INDEX_FILENAME = os.path.join(os.path.dirname(__file__), 'static-index.json')


class S3HashedFilesStorage(CachedFilesMixin, S3BotoStorage):
    """
    A bit of a hack to prevent S3BotoStorage + cached files from actually
    requesting files from S3 when a new process starts up -- which can take
    several seconds.
    
    Instead, during `collectstatic` (in post_process) we build a JSON file
    to use as a map from static asset name -> final serving URL (on S3).
    This file is saved and deployed with the app. When the process starts,
    it reads that file once and keeps it in memory.
    
    (Note that memcache is not a good solution for this case -- we want
    each worker to serve the copy of JS/CSS/etc. that goes with the
    HTML it is serving, rather than serving what is freshest in memcache.
    Also making a memcache hit for every static resource seems silly.)
    """
    _index_dict = None
    _building_index = False

    def url_from_index(self, name):
      if self._index_dict is None:
        try:
          with open(INDEX_FILENAME, 'r') as index_file:
            self._index_dict = json.load(index_file)
          logging.info('Loaded static index file (%d items)', len(self._index_dict))
        except IOError:
          self._index_dict = {}
          if not settings.DEBUG:
            logging.warn('Failed to load index file.')

      url = self._index_dict.get(name)
      if not url and not settings.DEBUG:
        logging.warn('Could not find static file: %s', name)
      return url

    def url(self, name, force=False):
      url = None

      if not self._building_index:
        url = self.url_from_index(name)

      if not url or force:
        url = super(S3HashedFilesStorage, self).url(name, force=force)

      return url

    def post_process(self, paths, dry_run=False, **options):
      result = super(S3HashedFilesStorage, self).post_process(paths, dry_run=dry_run, **options)

      # During a dry run, don't actually build the JSON index. This makes a
      # lot of sense, but one extra reason is that files aren't actually copied
      # to S3 during a dry run so S3BotoStorage won't include them yet and then
      # the below (self.url(path)) explodes.
      if not dry_run:
        # Prevent url() from trying to read the existing json file, which we
        # are right in the middle of building.
        self._building_index = True

        paths_to_url = {}
        for path in paths.keys():
          paths_to_url[path] = self.url(path)

        with open(INDEX_FILENAME, 'w+') as index_file:
          json.dump(paths_to_url, index_file)

        self._building_index = False

      return result

Updated to fix an issue with new files + dry run.