Print a table with enabled nginx modules https://wiki.debian.org/Nginx

modules.py

#!/usr/bin/env python
#
# Prints information about modules, whether these are enabled or not.
# The goal is to quickly find whether a module appears in some package
# variant for some distro version.
#
# Written for use in https://wiki.debian.org/Nginx
#
# The resulting table should appear as:
#
# |       || wheezy (1.2.x) || wheezy (1.6.x)       |
# |       || light | full   || light | full | other |
# | mod1  || yes   | no     || yes   | no   | no    |
# | mod2  || yes   | no     || yes   | no   | no    |
#
# Copyright (c) 2015 Peter Wu <[email protected]>
# License: MIT (compatible with BSD-2-Clause)
#

"""
Generate ngx_modules.c files with auto/configure. On Debian packaging, use:

apt-get -t wheezy build-dep nginx
apt-get -t wheezy source nginx
debian/rules config.status.{light,full,extras,naxsi}
for i in light naxsi full extras; do grep -Po 'ngx_\K\S+(?=_module;)' debian/build-$i/objs/ngx_modules.c | sed "s/^/nginx-$i /"; done > modules-1.2

apt-get -t wheezy-backports build-dep nginx
apt-get -t wheezy-backports source nginx
debian/rules override_dh_auto_configure
for i in light full extras; do grep -Po 'ngx_\K\S+(?=_module;)' debian/build-$i/objs/ngx_modules.c | sed "s/^/nginx-$i /"; done > modules-1.6

Then generate the table output with:

    ./modules.py modules-1.6 modules-1.2
"""

import sys
from collections import OrderedDict

# Internal modules
modules_ignore = [
    'conf',
    'core',
    'epoll',
    'errlog',
    'event_core',
    'events',
    'http',
    'http_chunked_filter', # Replaces http_chunkin_filter
    'http_copy_filter',
    'http_header_filter',
    'http_not_modified_filter',
    'http_postpone_filter',
    'http_spdy',
    'http_range_body_filter',
    'http_range_header_filter',
    'http_static',
    'http_write_filter',
    'mail',
    'openssl',
    'regex',
    # variants for http_upstream (can disable with --without-..._module)
    'http_upstream_ip_hash',
    'http_upstream_keepalive',
    'http_upstream_least_conn',
    # paid modules (documented but not available, internal name might differ)
    #'http_f4f',
    #'http_hls',
    #'http_session_log',
    #'http_status',
    #'stream_core',
    #'stream_proxy',
    #'stream_upstream',
]

# Maps internal module names to "Friendly" human-readable names (overrides
# guessed names in module_to_displayname).
module_friendly_name = {
    'http_gzip_static':     'Gzip Precompression',
    'http_realip':          'Real IP',
    'http_autoindex':       'Auto Index',
    'http_fastcgi':         'FastCGI',
    'http_upstream_fair':   'Upstream Fair Queue',
    'http_geoip':           'GeoIP',
    'http_limit_req':       'Limit Requests',
    'http_limit_conn':      'Limit Connections',
    'http_sub_filter':      'Substitution',
    'http_dav':             'WebDAV',
    'http_userid_filter':   'User ID',
    'http_perl':            'Embedded Perl',

    'mail_core':            'Mail Core',

    # 3rd party modules
    'http_subs_filter':     'HTTP Substitutions',
    'http_fancyindex':      'Fancy Index',
    'http_uploadprogress':  'Upload Progress',
    'http_push':            'HTTP Push',
    'http_lua':             'Embedded Lua',
    'development-kit':      'Nginx Development Kit',
}

def module_to_displayname(name):
    if name in module_friendly_name:
        return module_friendly_name[name]

    parts = name.split('_')

    # Drop common prefix
    if parts[0] in ('http', 'mail'):
        parts = parts[1:]

    # Drop "filter" suffix
    if len(parts) and parts[-1] == 'filter' and name != 'http_image_filter':
        parts = parts[:-1]

    def fixword(word):
        """Returns the word in the most appropariate (capitalized) form."""
        if word in ('http', 'dav', 'flv', 'mp4', 'scgi', 'ssi', 'uwsgi',
            'xslt', 'gif',
            'ssl', 'imap', 'pop3', 'smtp', 'pam'):
            return word.upper()

        return word.capitalize()

    name = ' '.join(fixword(word) for word in parts)
    return name

def main_module_to_url(name):
    # Converts a name from modules_main to the name as used in the docs.
    # The internal name differs from the doc name for some filter mods.
    if name.endswith('_filter') and name != 'http_image_filter':
        name = name.replace('_filter', '')

    return 'http://nginx.org/en/docs/http/ngx_{}_module.html'.format(name)

# Documented modules (minus the paid modules in modules_ignore). Generated by:
#   curl -s http://nginx.org/en/docs/ |
#   grep -Po '^ngx_\K\S+(?=_module)' | sed "s/.*/    '&',/"
modules_main = OrderedDict([ (name, main_module_to_url(name)) for name in [
    'http_core',
    'http_access',
    'http_addition_filter',
    'http_auth_basic',
    'http_auth_request',
    'http_autoindex',
    'http_browser',
    'http_charset_filter',
    'http_dav',
    'http_empty_gif',
    'http_fastcgi',
    'http_flv',
    'http_geo',
    'http_geoip',
    'http_gunzip_filter',
    'http_gzip_filter',
    'http_gzip_static',
    'http_headers_filter',
    'http_image_filter',
    'http_index',
    'http_limit_conn',
    'http_limit_req',
    'http_log',
    'http_map',
    'http_memcached',
    'http_mp4',
    'http_perl',
    'http_proxy',
    'http_random_index',
    'http_realip',
    'http_referer',
    'http_rewrite',
    'http_scgi',
    'http_secure_link',
    'http_spdy_filter',
    'http_split_clients',
    'http_ssi_filter',
    'http_ssl',
    'http_stub_status',
    'http_sub_filter',
    'http_upstream',
    'http_userid_filter',
    'http_uwsgi',
    'http_xslt_filter',

    'mail_core',
    'mail_auth_http',
    'mail_proxy',
    'mail_ssl',
    'mail_imap',
    'mail_pop3',
    'mail_smtp',
]])

# additional external modules with documentation links
modules_additional = OrderedDict([
# Documentation links for external modules
    ('http_auth_pam',           'https://github.com/stogh/ngx_http_auth_pam_module'),
    ('http_cache_purge',        'https://github.com/FRiCKLE/ngx_cache_purge'),
    # Note, chunkin is replaced by chunked core module
    ('http_chunkin_filter',     'https://github.com/agentzh/chunkin-nginx-module'),
    ('http_dav_ext',            'https://github.com/arut/nginx-dav-ext-module'),
    ('http_echo',               'https://github.com/openresty/echo-nginx-module'),
    ('http_fancyindex',         'https://github.com/aperezdc/ngx-fancyindex'),
    ('http_headers_more_filter', 'https://github.com/openresty/headers-more-nginx-module'),
    ('http_lua',                'https://github.com/openresty/lua-nginx-module'),
    ('http_naxsi',              'https://github.com/nbs-system/naxsi'),
    ('http_push',               'https://github.com/slact/nginx_http_push_module'),
    ('http_subs_filter',        'https://github.com/yaoweibin/ngx_http_substitutions_filter_module'),
    ('http_upload',             'https://github.com/vkholodkov/nginx-upload-module/tree/2.2'),
    ('http_uploadprogress',     'https://github.com/masterzen/nginx-upload-progress-module'),
    ('http_upstream_fair',      'https://github.com/gnosek/nginx-upstream-fair'),
    # Not a user-visible module, enabled for 'extras'
    ('development-kit',         'https://github.com/simpl/ngx_devel_kit'),
])

def print_modules(enabled_modules, title, registered_modules):
    # Map from version to available package variants
    versions = OrderedDict()
    for versions_to_variants in enabled_modules.values():
        pass

    # Print header (skip module name)
    pkg_row = '||<|2> '
    variants_row = ''
    for version, variants in enabled_modules.items():
        # +2 for package version and variant names
        pkg_row += '||<|{}> '.format(2 + len(registered_modules.keys()))

        # the package (distro) version
        pkg_row += '||<-{}:> {} '.format(len(variants.keys()), version)
        # The package variants (light, etc.)
        for variant in variants.keys():
            variants_row += '||<:> {} '.format(variant)
    pkg_row += '||'
    variants_row += '||'

    cols_count = sum(1 + len(vs.keys()) for vs in enabled_modules.values())
    print("||<:-{}> ''' {} ''' ||".format(cols_count, title))
    print(pkg_row)
    print(variants_row)

    for name, url in registered_modules.items():

        # Add the module name to the table
        display_name = module_to_displayname(name)
        if url:
            cell = '[[{}|{}]]'.format(url, display_name)
        else:
            cell = display_name
        row = '|| ' + cell + ' '

        # Print enabled states for each variant
        for variants in enabled_modules.values():
            #row += '|| ' # already rowspanned
            for variant in variants.values():
                # Whether the module name is enabled in this variant
                if name in variant:
                    row += '||<:#80FF80> OK '
                else:
                    row += '||<:#FF8080> NO '

        row += '||'
        print(row)

# Names of all modules
modules_all = list(modules_main.keys()) + list(modules_additional.keys())
modules_all += modules_ignore

# Find all enabled modules
# Input format: variant [SPC] modname. Ex: light http_access
def parse_file(f):
    modules_variants_enabled = OrderedDict()
    for line in f:
        variant, name = line.strip().split()
        if not name in modules_all:
            sys.stderr.write('Unknown name {}\n'.format(name))
            continue

        # Maintain variant order in file, add a new set to hold enabled modules
        if variant not in modules_variants_enabled:
            modules_variants_enabled[variant] = set()

        modules_variants_enabled[variant].add(name)
    if 'extras' in modules_variants_enabled:
        modules_variants_enabled['extras'].add('development-kit')
    return modules_variants_enabled

def filename_to_version(filename):
    """Given a filename, tries to make a human-readable label."""
    if filename == 'modules-1.2':
        return 'squeeze-backports / wheezy (1.2.1-2.2)'
    if filename == 'modules-1.6':
        return 'wheezy-backports / jessie (1.6.2-5)'
    if filename == 'modules-1.6.2-9':
        return 'wheezy-backports / jessie (1.6.2-9)'
    return filename

def main():
    # Maps distro versions to a mapping from package variants to a set of
    # modules enabled in that variant.
    enabled_modules = OrderedDict()
    for filename in sys.argv[1:]:
        name = filename_to_version(filename)

        # Do not overwrite existing entries...
        if name in enabled_modules:
            name = filename

        with open(filename) as f:
            enabled_modules[name] = parse_file(f)

    #import pprintl; pprint.pprint(enabled_modules)

    def filter_mods(d, prefix):
        return OrderedDict([ (name, value)
            for name, value in d.items()
                if name.startswith(prefix)
        ])
    print_modules(enabled_modules, 'HTTP Modules', filter_mods(modules_main, 'http_'))
    print_modules(enabled_modules, 'Mail Modules', filter_mods(modules_main, 'mail_'))
    print_modules(enabled_modules, 'Third Party Modules', modules_additional)

if __name__ == '__main__':
    main()

print-human-names.py

#!/usr/bin/env python
#
# Prints the human-readable names for various modules.
#
# Usage:
#
#   ./print-human-names.py modules-file [package-name]
#   modules-file is the same file as for modules.py
#   Outputs data suitable for a control file.
#
#   ./print-human-names.py modules-file package-name existing-file
#   existing-file is a file with the names of existing human-readable text per
#   line. The purpose of this mode is to find missing names in the control file.
#
# Copyright (c) 2015 Peter Wu <[email protected]>
# License: MIT (compatible with BSD-2-Clause)
#

import sys, textwrap
from itertools import chain
from collections import OrderedDict
from modules import modules_main, modules_additional, \
    parse_file, module_to_displayname

def print_table(rows):
    zip(*rows)
    format_string = ' '.join('{{:{}}}'.format(
        max(1, max(len(str(c)) for c in cells))
    ) for cells in zip(*rows))
    for row in rows:
        print(format_string.format(*[str(c) for c in row]))


def filter_main_modules(fn):
    return list(filter(fn, modules_main))

# Modules which are enabled by default
# auto/configure --help |
# grep -Poe '--without-\K(http|mail)[a-z0-9_]+(?=_module)' | sed "s/.*/'&',/"
modules_main_default = [
    # not included with the grep output
    'http_core',

    'http_charset', 'http_gzip', 'http_ssi', 'http_userid', 'http_access',
    'http_auth_basic', 'http_autoindex', 'http_geo', 'http_map',
    'http_split_clients', 'http_referer', 'http_rewrite', 'http_proxy',
    'http_fastcgi', 'http_uwsgi', 'http_scgi', 'http_memcached',
    'http_limit_conn', 'http_limit_req', 'http_empty_gif', 'http_browser',
    'http_upstream_hash', 'http_upstream_ip_hash', 'http_upstream_least_conn',
    'http_upstream_keepalive', 'mail_pop3', 'mail_imap', 'mail_smtp',
]


cats = [
    ('Standard HTTP',
        filter_main_modules(lambda name:
            name.startswith('http_') and name in modules_main_default)),
    ('Optional HTTP',
        filter_main_modules(lambda name:
            name.startswith('http_') and name not in modules_main_default)),
    ('Mail',
        filter_main_modules(lambda name: name.startswith('mail_'))),
    ('Third Party', modules_additional),
]

def print_control(mods):
    wrapper = textwrap.TextWrapper(79, initial_indent=' ', subsequent_indent=' ')

    for variant, enabled_modules in mods.items():
        print('Package: {}'.format(variant))
        for title, modules in cats:
            if not any(name in enabled_modules for name in modules):
                continue
            text = '{} modules: '.format(title).upper()
            text += ', '.join(module_to_displayname(name)
                for name in modules if name in enabled_modules)
            text += '.'
            text = '\n'.join(wrapper.wrap(text))
            print(' .')
            print(text)
        print('')

def print_for_type(names_filename, enabled_modules):
    read_displaynames = []
    with open(names_filename) as f:
        for line in f:
            line = line.strip()
            read_displaynames.append(line)

    for title, modules in cats:
        if not any(name in enabled_modules for name in modules):
            continue
        print('{} modules:'.format(title).upper())
        rows = []
        for name in modules:
            if name not in enabled_modules:
                continue
            dname = module_to_displayname(name)
            rname = dname in read_displaynames
            if rname is True:
                rname = ''
            rows.append((name, dname, rname))

        print_table(rows)
        print('')

if __name__ == '__main__':
    with open(sys.argv[1]) as f:
        mods = parse_file(f)
        limit_version = sys.argv[2] if len(sys.argv) > 2 else None

        if len(sys.argv) <= 3:
            if limit_version is not None:
                mods = { k: v for k, v in mods.items() if k == limit_version }
            print_control(mods)
        else:
            assert limit_version in mods, '{} not found'.format(limit_version)
            print_for_type(sys.argv[3], mods[limit_version])