Finds the worst includes in a C++ source tree

analyzeincludes.py

import subprocess
import contextlib
import os
import sys
import fnmatch
from collections import defaultdict, Counter

@contextlib.contextmanager
def chdir(dirname=None):
    curdir = os.getcwd()
    try:
        if dirname is not None:
            os.chdir(dirname)
            yield
    finally:
        os.chdir(curdir)


def count_includes(data):
    indirect_includes = defaultdict(set)
    direct_includes = set()

    cur_include = None
    for line in data.split("\n"):
        try:
            depth_dots, raw_path = line.split(" ")
        except ValueError:
            break

        if os.path.realpath(raw_path).startswith("/usr"):
            continue
        path = os.path.relpath(raw_path, rts_dir)

        if len(depth_dots) == 1:
            cur_include = path
            direct_includes.add(path)
        else:
            indirect_includes[cur_include].add(path)

    return direct_includes, indirect_includes

rts_dir = os.getcwd()
lib_dir = rts_dir + "/lib"
inc_dir = rts_dir + "/../include"

includes = [
    "/usr/include/AL",
    "/usr/include/freetype2",
    rts_dir,
    lib_dir,
    inc_dir + "/SDL",
    lib_dir + "/lua/include",
    lib_dir + "/assimp/include",
]

base_args = ["g++", "-H", "-E", "-pipe"]

for include in includes:
    base_args.append("-I" + include)

def main():
    direct_inc_counter = Counter()
    indirect_inc_counter = Counter()
    max_indirect = defaultdict(int)

    sys.stderr.write("running '%s' on all files\n" % " ".join(base_args))

    try:
        for dirpath, dirnames, filenames in os.walk('.'):
            try:
                dirnames.remove("lib")
            except ValueError:
                pass
            try:
                dirnames.remove("Win")
            except ValueError:
                pass
            try:
                dirnames.remove("Mac")
            except ValueError:
                pass

            for filename in fnmatch.filter(filenames, '*.cpp'):
                with chdir(dirpath):
                    args = base_args + [filename]
                    cxx = subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, universal_newlines=True)
                    result = cxx.stderr.read()
                    cxx.wait(timeout=1)
                    if cxx.returncode == 0:
                        sys.stderr.write("Processed %s/%s\n" % (dirpath, filename))
                    else:
                        sys.stderr.write("Failed %s/%s with %s:\n%s"  % (dirpath, filename, args, result))

                    direct_includes, indirect_includes = count_includes(result)
                    direct_inc_counter.update(direct_includes)
                    for primary_include, sub_includes in indirect_includes.items():
                        indirect_inc_counter[primary_include] += len(sub_includes)
                        max_indirect[primary_include] = max(max_indirect[primary_include], len(sub_includes))
    finally:
        total_inclusions = direct_inc_counter + indirect_inc_counter
        print("File accesses\tSub-includes\tUses\tMax sub-includes\tHeader")
        data = [(count, indirect_inc_counter[header], direct_inc_counter[header], max_indirect[header], header) for header, count in total_inclusions.most_common()]
        data.sort(reverse=True)
        for count, indirect, direct, max_indirect_, header in data:
            print("%d\t\t%d\t\t%d\t%d\t\t\t%s" % (count, indirect, direct, max_indirect_, header))

if __name__ == "__main__":
    main()