Benchmark for tokenizing a Django Template.

gistfile1.txt

import timeit

crispy = """{% load crispy_forms_field %}

{% if field.is_hidden %}
    {{ field }}
{% else %}
    {% if field|is_checkbox and tag != "td" %}
        <div class="mb-3{% if 'form-horizontal' in form_class %} row{% endif %}">
        {% if label_class %}
            <div class="{% for offset in bootstrap_checkbox_offsets %}{{ offset|slice:"7:14" }}{{ offset|slice:"4:7" }}{{ offset|slice:"14:16" }} {% endfor %}{{ field_class }}">
        {% endif %}
    {% endif %}
    <{% if tag %}{{ tag }}{% else %}div{% endif %} id="div_{{ field.auto_id }}" class="{% if field|is_checkbox and form_show_labels %}form-check{% else %}mb-3{% if 'form-horizontal' in form_class %} row{% endif %}{% endif %}{% if wrapper_class %} {{ wrapper_class }}{% endif %}{% if field.css_classes %} {{ field.css_classes }}{% endif %}">
        {% if field.label and not field|is_checkbox and form_show_labels %}
            {% if field.use_fieldset %}<fieldset{% if 'form-horizontal' in form_class %} class="row"{% endif %}{% if field.aria_describedby %} aria-describedby="{{ field.aria_describedby }}"{% endif %}>{% endif %}
            <{% if field.use_fieldset %}legend{% else %}label{% endif %}
                {% if field.id_for_label %}for="{{ field.id_for_label }}"{% endif %} class="{% if 'form-horizontal' in form_class %}col-form-label pt-0{% else %}form-label{% endif %}{% if label_class %} {{ label_class }}{% endif %}{% if field.field.required %} requiredField{% endif %}{% if field.use_fieldset %} fs-6{% endif %}">
                {{ field.label }}{% if field.field.required %}<span class="asteriskField">*</span>{% endif %}
            </{% if field.use_fieldset %}legend{% else %}label{% endif %}>
        {% endif %}

        {% if field|is_checkboxselectmultiple or field|is_radioselect %}
            {% include 'bootstrap5/layout/radio_checkbox_select.html' %}
        {% endif %}

        {% if not field|is_checkboxselectmultiple and not field|is_radioselect %}
            {% if field|is_checkbox and form_show_labels %}
                    {% if field.errors %}
                        {% crispy_field field 'class' 'form-check-input is-invalid' %}
                    {% else %}
                        {% crispy_field field 'class' 'form-check-input' %}
                    {% endif %}
                <label for="{{ field.id_for_label }}" class="form-check-label{% if field.field.required %} requiredField{% endif %}">
                    {{ field.label }}{% if field.field.required %}<span class="asteriskField">*</span>{% endif %}
                </label>
                {% include 'bootstrap5/layout/help_text_and_errors.html' %}
            {% else %}
                {% if field_class %}<div class="{{ field_class }}">{% endif %}
                    {% if field|is_file %}
                        {% include 'bootstrap5/layout/field_file.html' %}
                    {% elif field|is_select %}
                        {% if field.errors %}
                            {% crispy_field field 'class' 'form-select is-invalid' %}
                        {% else %}
                            {% crispy_field field 'class' 'form-select' %}
                        {% endif %}
                    {% elif field|is_checkbox %}
                        {% if field.errors %}
                            {% crispy_field field 'class' 'form-check-input is-invalid' %}
                        {% else %}
                            {% crispy_field field 'class' 'form-check-input' %}
                        {% endif %}
                    {% elif field.errors %}
                        {% crispy_field field 'class' 'form-control is-invalid' %}
                    {% else %}
                        {% crispy_field field 'class' 'form-control' %}
                    {% endif %}
                    {% if not field|is_file %}
                        {% include 'bootstrap5/layout/help_text_and_errors.html' %}
                    {% endif %}
                {% if field_class %}</div>{% endif %}
            {% endif %}
        {% endif %}
        {% if field.use_fieldset and field.label and form_show_labels %}</fieldset>{% endif %}
    </{% if tag %}{{ tag }}{% else %}div{% endif %}>
    {% if field|is_checkbox and tag != "td" %}
        {% if label_class %}
            </div>
        {% endif %}
        </div>
    {% endif %}
{% endif %}"""

admin_index = """{% extends "admin/base_site.html" %}
{% load i18n static %}

{% block extrastyle %}{{ block.super }}<link rel="stylesheet" href="{% static "admin/css/dashboard.css" %}">{% endblock %}

{% block coltype %}colMS{% endblock %}

{% block bodyclass %}{{ block.super }} dashboard{% endblock %}

{% block nav-breadcrumbs %}{% endblock %}

{% block nav-sidebar %}{% endblock %}

{% block content %}
<div id="content-main" class="app-list">
  {% include "admin/app_list.html" with app_list=app_list show_changelinks=True %}
</div>
{% endblock %}

{% block sidebar %}
<div id="content-related">
    <div class="module" id="recent-actions-module">
        <h2>{% translate 'Recent actions' %}</h2>
        <h3>{% translate 'My actions' %}</h3>
            {% load log %}
            {% get_admin_log 10 as admin_log for_user user %}
            {% if not admin_log %}
            <p>{% translate 'None available' %}</p>
            {% else %}
            <ul class="actionlist">
            {% for entry in admin_log %}
            <li class="{% if entry.is_addition %}addlink{% endif %}{% if entry.is_change %}changelink{% endif %}{% if entry.is_deletion %}deletelink{% endif %}">
                <span class="visually-hidden">{% if entry.is_addition %}{% translate 'Added:' %}{% elif entry.is_change %}{% translate 'Changed:' %}{% elif entry.is_deletion %}{% translate 'Deleted:' %}{% endif %}</span>
                {% if entry.is_deletion or not entry.get_admin_url %}
                    {{ entry.object_repr }}
                {% else %}
                    <a href="{{ entry.get_admin_url }}">{{ entry.object_repr }}</a>
                {% endif %}
                <br>
                {% if entry.content_type %}
                    <span class="mini quiet">{% filter capfirst %}{{ entry.content_type.name }}{% endfilter %}</span>
                {% else %}
                    <span class="mini quiet">{% translate 'Unknown content' %}</span>
                {% endif %}
            </li>
            {% endfor %}
            </ul>
            {% endif %}
    </div>
</div>
{% endblock %}"""

if __name__ == "__main__":
    n_exec = 100
    n_nested = 2000
    template_str_cases = [
        ("crispy", crispy),
        ("admin_index", admin_index),
    ]

    for case_name, template_str in template_str_cases:
        # Original approach - tokenize every time
        time_original = timeit.timeit(
            "Lexer('''" + template_str + "''').tokenize()",
            setup="from django.template.base import Lexer",
            number=n_exec,
        )
        t_per_exec_original = time_original / n_exec

        # Optimized approach - pre-tokenize and reuse
        time_optimized = timeit.timeit(
            "tokens",
            setup="""
from django.template.base import Lexer
template_str = '''"""
            + template_str
            + """'''
tokens = Lexer(template_str).tokenize()
            """,
            number=n_exec,
        )
        t_per_exec_optimized = time_optimized / n_exec

        print("----------------\nCASE: {}".format(case_name))
        print("Original approach:")
        print(f"Total time taken: {time_original:.6f} seconds")
        print(f"Time per execution: {t_per_exec_original:.6f} seconds")
        print("\nOptimized approach (pre-tokenized):")
        print(f"Total time taken: {time_optimized:.6f} seconds")
        print(f"Time per execution: {t_per_exec_optimized:.6f} seconds")
        print(f"Speedup factor: {t_per_exec_original / t_per_exec_optimized:.2f}x")