Python multiprocess logging

multiprocess_logging_example.py

from contextlib import contextmanager
import multiprocessing


def main():
    multiprocessing.current_process().name = 'master'
    mgr = multiprocessing.Manager()

    with setup_logging(mgr):
        pass # do cool stuff here


@contextmanager
def setup_logging(manager, verbose=True):
    import logging
    from logging.handlers import QueueHandler, QueueListener

    fmt = '%(asctime)s [%(process)2d:%(processName)-11s] %(name)s %(levelname)5s: %(message)s'

    class CustomQueueHandler (QueueHandler):
        def enqueue(self, record):
            self.queue.put(record)

    logging_queue = manager.Queue()

    logging.getLogger('').setLevel(logging.DEBUG)

    qh = CustomQueueHandler(logging_queue)
    logging.getLogger('').addHandler(qh)

    handlers = []
    h = logging.StreamHandler()
    h.setLevel(logging.DEBUG if verbose else logging.INFO)
    h.setFormatter(logging.Formatter(fmt))
    handlers.append(h)

    # Create FileHandler here and append it to handlers if needed

    ql = QueueListener(logging_queue, *handlers)
    ql.start()
    try:
        yield
    finally:
        ql.stop()