Showing usused views in Django by parsing nginx acces log

django_unused_views.py

SKIP_MODULES = ('django', 'simple_history')
ACCESS_FILES = [
    '/var/log/nginx/access.log',
    '/var/log/nginx/access.log.1',
    '/var/log/nginx/access.log.2',
] + ['/var/log/nginx/access.log.{}.gz'.format(i) for i in range(3, 31)]
RE_SEARCH_URL = r'(?:(?:POST)|(?:GET)|(?:PATCH)|(?:PUT)|(?:OPTIONS)|(?:DELETE)|(?:HEAD))\s([^\s\?]+)'
def SKIP_MATCH(url):
    return requested_url.startswith('/static/')

import re
import gzip
from django.urls import get_resolver
from django.urls.exceptions import Resolver404
from django.urls import resolve

def get_callbacks(url_patterns):
    for pattern in url_patterns:
        if not pattern.callback:
            yield from get_callbacks(pattern.url_patterns)
        else:
            if pattern.callback.__module__.split('.')[0] in SKIP_MODULES:
                continue
            yield pattern.callback
all_funcs = set(list(get_callbacks(get_resolver().url_patterns)))
print('Total Functions: {}'.format(len(all_funcs)))

for f_name in ACCESS_FILES:
    with (gzip.open(f_name) if f_name.endswith('.gz') else open(f_name)) as fh:
        for line in fh:
            if isinstance(line, bytes):
                line = line.decode('utf-8')
            search_url = re.search(RE_SEARCH_URL, line)
            if not search_url:
                continue
            requested_url = search_url[1]
            if SKIP_MATCH(requested_url):
                continue
            try:
                requested_func = resolve(requested_url).func
            except Resolver404:
                continue

            try:
                all_funcs.remove(requested_func)
            except KeyError:
                pass
        print('Left {} after {}'.format(len(all_funcs), f_name))
        
def get_patern_by_func(func, url_patterns, modules=None):
    if modules is None:
        modules = []
        
    for pattern in url_patterns:
        
        if pattern.callback:
            if pattern.callback == func:
                return (pattern, modules)
        else:
            mod_ret = get_patern_by_func(func, pattern.url_patterns, modules + [pattern])
            if mod_ret:
                return mod_ret
            
for func in all_funcs:
    pattern, modules = get_patern_by_func(func, get_resolver().url_patterns)
    print(func)
    print(pattern)
    print(modules)
    print('-'*10)