Jinja2 compiled templates module loader for App Engine Pyhton 2.7.

compiler.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, with_statement 

import logging
import zipfile
import StringIO
import marshal
import imp
import struct
import time
from jinjaenv import models, jinja2env


def code_gen(entity, env=None):
    """ Jinja module code """

    eid = entity.key.id()
    logging.info('compiled : ' + eid)
    name = eid.split('.')[0]

    raw = env.compile(entity.content, name=name, filename=name + '.html', raw=True)
    i = 0
    py_code = ''
    indentation = " " * 4
    for line in raw.split("\n"):
        if i < 2:
            py_code += (line + "\n")
        else:
            if i == 2:
                py_code += "__jinja_template__ = None\n"
                py_code += "def run(environment, jinja_template):\n"
                py_code += indentation + "global __jinja_template__\n"
                py_code += indentation + "__jinja_template__ = jinja_template\n"
            py_code += (indentation + line + "\n")
        i += 1

    py_code += indentation + "return (name, blocks, root, debug_info)\n"
    return py_code


def compile_callback(entity, env):
    """ NDB map async callback """

    py_code = code_gen(entity, env)
    code_object = compile(py_code.encode(), ('%s.%s' % (entity.key.parent().id(), 
                                                        entity.key.id())).encode(), 'exec')
    entity.blob = marshal.dumps(code_object)
    return entity.put_async()


def compile_runtimes(site_key, env):
    """ Compile runtimes HTML and TXT templates into Python code objects """

    logging.info('compiling runtime html templates')
    env = jinja2env.JinjaEnv.get_env(site_key.id())
    query = models.Runtimes.query(ancestor=site_key)
    future = query.map_async(lambda entity: compile_callback(entity, env))
    future.get_result()


def zip_compiled(site_key, response):
    """ Download zip Runtimes compiled Python files : .pyc """

    updates = []
    env = jinja2env.JinjaEnv.get_env(site_key.id())
    output = StringIO.StringIO()

    with zipfile.ZipFile(output, 'w') as z:

        for each in models.RunTimes.query(ancestor=site_key)
            member_name = site_key.id() + '.' + each.key.id().split('.')[0] + '.pyc'
            # compile again, to make sure we have the latest version
            code_object = compile(code_gen(each, env).encode(),
                                 ('%s.%s' % (site_key.id(), each.key.id())).encode(), 'exec')
            each.blob = marshal.dumps(code_object)
            updates.append(each)
            timestamp = struct.pack(b'f', time.mktime(each.modified.timetuple()))
            z.writestr(member_name.encode(), imp.get_magic() + timestamp + each.blob)

    ndb.put_multi(updates)
    response.headers[b'Content-Type'] = b'multipart/x-zip'
    response.headers[b'Content-Disposition'] = str('attachment; filename=%s-compiled-templates.zip'
                                                   % site_key.id())
    response.out.write(output.getvalue())

Jinja module loader.md

Jinja compiled templates module loader

This code is part of a Jinja CMS for Google App Engine Python 2.7 and NDB datastore

A Jinja enviroment is created for every CMS site: site_key_id = 'example

The modules are created using compiler.py The resulting code objects are stored in the dadastore using Kind Runtimes and a BlobProperty

The modules can also be saved / downloaded as .pyc in a zip archive: <site_key_id>-compiled-templates.zip If this zip archive is part of an app engine project, the module loader does not need the Runtimes blobs to run the modules.

The zip archive folder name is part of the env_variables in the app.yaml : JINJACMS_LOADPY

class Runtimes(ndb.Model):
    site = ndb.ComputedProperty(lambda self: self.key.parent().id())
    content = ndb.TextProperty(default='')                # html or txt template content
    blob = ndb.BlobProperty(default=None)                 # code object
    modified = ndb.DateTimeProperty(auto_now=True)
    created = ndb.DateTimeProperty(auto_now_add=True)

moduleloader.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import imp
import sys
import os
from google.appengine.ext import ndb
import zipfile
import marshal
import logging
from jinja2 import Environment
from jinja2.exceptions import TemplateNotFound

#    Loading templates from Python modules from datastore or a zip archive in zip_dir
#    Based on : https://groups.google.com/group/pocoo-libs/browse_thread/thread/748b0d2024f88f64


def init_env(site_key_id):
    """ initialize Jinja environment : using the module loader
        JINJACMS_LOADPY in app.yaml env_variables, value : compiled or NONE
        zip archive with .pyc, name : <site_key_id>-compiled-templates.zip """

    if 'JINJACMS_LOADPY' not in os.environ or os.environ['JINJACMS_LOADPY'].lower() in ['', 'none']:
        path = None
        logging.info('%s RUNTIME_MODULE loader initialized, using Python code objects' % site_key_id)
    else:
        path = os.path.join(os.path.dirname(sys.modules['appengine_config'].__file__),
                            os.environ['JINJACMS_LOADPY'], site_key_id + '-compiled-templates.zip')
        logging.info('%s RUNTIME_MODULE loader initialized, using zip in %s' % (site_key_id, path))

    loader = FileSystemModuleLoader(site_key_id, path)
    return Environment(auto_reload=False, loader=loader)


class ModuleLoader(object):
    """Base mixin class for loaders that use pre-parsed Jinja2 templates stored as Python code. """

    def get_module(self, environment, template):
        raise TemplateNotFound(template)

    def load(self, environment, filename, j_globals=None):
        """ Loads a pre-compiled template, stored as Python code in a template module. """

        if j_globals is None:
            j_globals = {'environment': environment}

        t = object.__new__(environment.template_class)

        module = self.get_module(environment, filename)
        name, blocks, root, debug_info = module.run(environment, t)
        
        t.environment = environment
        t.globals = j_globals
        t.name = name
        t.filename = filename
        t.blocks = blocks
        # render function and module
        t.root_render_func = root
        t._module = None

        # debug and loader helpers
        t._debug_info = debug_info
        t._uptodate = lambda: True
        return t


class FileSystemModuleLoader(ModuleLoader):
    """ Load compiled Jinja templates from the datastore (code objects)
        or .py from a zip archive for the CMS sites(site_keys) """

    def __init__(self, site_key_id, path=None):

        self.site_key_id = site_key_id
        self.site_key = ndb.Key('CmsSites', site_key_id)

        if path:
            try:
                self._zf = zipfile.ZipFile(os.path.join(path), 'r')
            except IOError, e:
                logging.warning('Zip archive not found, path : %s. Using Runtimes datastore.'
                                ' Exception : %s' % (path, str(e)))
                self._zf = None
        else:
            self._zf = None

        # fake module : <site_key_id>
        mod = imp.new_module(site_key_id)
        mod.__loader__ = self
        mod.__file__ = "[fake module %r]" % site_key_id
        mod.__path__ = []
        sys.modules[site_key_id] = mod

    def get_module(self, environment, template):
        """ Convert the template to a module name and load the code """

        mod_name = '%s.%s' % (self.site_key_id, template.replace('.html', '')
                              .replace('.txt', '').replace('/', '.'))
        if mod_name in sys.modules:
            return sys.modules[mod_name]

        logging.info('load Jinja template module : ' + mod_name)
        try:
            if self._zf:
                # ignore first 8 bytes with magic and timestamp of .pyc
                module_code = marshal.loads(self._zf.read(mod_name + '.pyc')[8:])
            else:
                module_code = marshal.loads(ndb.Key('Runtimes', template,
                                                    parent=self.site_key).get().blob)
            module = imp.new_module(mod_name)
            exec module_code in module.__dict__
            sys.modules[mod_name] = module
            return module
        except (ImportError, AttributeError, KeyError):
            logging.error('load failed : ' + mod_name)
            raise TemplateNotFound(mod_name)