A simple way to log all indexes used by the application

hook_indexes.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = "Sahid Orentino Ferdjaoui"
__license__ = "Apache License, Version 2.0"

"""A simple way to log in the datastore all indexes used by your app
This module has 2 function:
  - The first is the hook, it run like a middleware with django.
  - The second is a view that simply the generation of the index.yaml file.


INSTALL:
For example if you put the module in your root directory at the name hook_indexes.py
# settings.py
MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    ...,
    'hook_indexes.IndexesMiddleware',
    ...
)
Now each times your datastore query needs an index, 
it will be recorded in the datastore on the model _hook_indexes_stats.

You can also configure a new view to generate all indexes in the index.yaml format.
For that you just need to add a new rule in your urls.py.
# urls.py
urlpatterns += patterns ('hook_indexes',
  (r'^private/hooks/indexes$', 'indexes_view'),
)

After that i it's recommended to secure this url.
# app.yaml
- url: /private/.*
  script: main.py
  login: admin
"""

import logging
import hashlib

from google.appengine.api import memcache
from google.appengine.api import apiproxy_stub_map
from google.appengine.ext import db
from google.appengine.datastore import datastore_index

from django.http import HttpResponse


LOCK_KEY="<l>"
LOCK_TIME=3600

FETCH_LIMIT=200 # number of authrized indexes.

class _hook_indexes_stats(db.Model):
    index = db.StringProperty(required=True, multiline=True)
    name = db.StringProperty(required=True)
    created = db.DateTimeProperty(auto_now_add=True)
    updated = db.DateTimeProperty(auto_now_add=True, auto_now=True)

class IndexesMiddleware (object):
    """Store indexes used by the application.

    Install:
      Just add this middleware in your django settings.
      MIDDLEWARE_CLASSES = (
        ...
        'hook_indexes.IndexesMiddleware',
        ...
      )
    """
    def process_request (self, request):
        def log_api_call(service, call, request, response):
            if call == 'RunQuery':
                required, kind, ancestor, props, eq_filters = \
                          datastore_index.CompositeIndexForQuery(request)

                index = datastore_index.IndexYamlForQuery(kind, ancestor, props)
                index_key = hashlib.sha1(index).hexdigest()

                if not memcache.get(index_key):
                    if memcache.add(index_key, LOCK_KEY, LOCK_TIME):
                        logging.info("indexes: \n%s", index)
                        _hook_indexes_stats(
                            key_name=index_key,
                            name=kind,
                            index=index).put()
        
        apiproxy_stub_map.apiproxy.GetPreCallHooks().Append(
            'log_api_call', log_api_call, 'datastore_v3')


TITLE_PATTERN = """
###
### %s
###
"""
def indexes_view(request):
    """Prints all indexes in the indexes.yaml format.

    To install this view, add a new entry in the urls.py:
        urlpatterns += patterns ('hook_indexes',
          (r'^private/hooks/indexes$', 'indexes_view'),
        )
    """
    indexes = _hook_indexes_stats.all().fetch(limit=FETCH_LIMIT)

    categorized = {}
    for ref in indexes:
        if not (ref.name in categorized):
            categorized[ref.name] = []
        categorized[ref.name].append(ref.index)

    response = []
    for name, indexes in categorized.items():
        response.append(TITLE_PATTERN % name)
        response.extend(["%s\n\n" % index for index in indexes])

    return HttpResponse("".join(response), mimetype="text/plain")