pglogslim.py

#! /usr/bin/env python3
# filter postgres logs to remove parameters of very large queries
#
# https://gist.github.com/millerdev/def20a42d78717174cd5722a9de2055a
import gzip
import re
import signal
import sys

# HACK prevent "BrokenPipeError: [Errno 32] Broken pipe" output on stderr
signal.signal(signal.SIGPIPE, lambda n, f: sys.exit())

PARAM_RE = re.compile(r"(\{.+?\}|ARRAY\[.+?\])")
WHERE_RE = re.compile(
    r'(OR \(""locations_sqllocation""\.""rght"" <= \d+ '
    r'AND ""locations_sqllocation""\.""lft"" >= \d+ '
    r'AND ""locations_sqllocation""\.""tree_id"" = \d+\) ?)+'
)


def param_summary(match):
    return "{%s values}" % (match.group(0).count(",") + 1)


def main():
    if len(sys.argv) == 1:
        msg = "Usage: {} (LOG_FILE | -) [LOG_FILE ...] > NEW_LOG_FILE\n"
        sys.stderr.write(msg.format(*sys.argv))
        sys.exit()

    for name in sys.argv[1:]:
        sys.stderr.write("processing {}\n".format(name))
        if name == '-':
            process_log(sys.stdin)
        elif name.endswith(".gz"):
            with gzip.open(name, mode='rt', encoding="utf-8") as log:
                process_log(log)
        else:
            with open(name, encoding="utf-8") as log:
                process_log(log)


def process_log(log):
    for line in log:
        if len(line) > 600:
            line = PARAM_RE.sub(param_summary, line)
            line = WHERE_RE.sub("OR ...", line)
        sys.stdout.write(line)


if __name__ == "__main__":
    main()