Python 3 logging configuration using JSON

logging2.json

{
    "logging": {
        "version": 1,
        "disable_existing_loggers": true,
        "formatters": {
            "brief": {
                "class": "logging.Formatter",
                "datefmt": "%I:%M:%S",
                "format": "%(levelname)-8s; %(name)-15s; %(message)s"
            },
            "single-line": {
                "class": "logging.Formatter",
                "datefmt": "%I:%M:%S",
                "format": "%(levelname)-8s; %(asctime)s; %(name)-15s; %(pathname)s:%(funcName)s;%(lineno)d: %(message)s"
            },
            "multi-process": {
                "class": "logging.Formatter",
                "datefmt": "%I:%M:%S",
                "format": "%(levelname)-8s; [%(process)d]; %(name)-15s; %(pathname)s:%(funcName)s;%(lineno)d: %(message)s"
            },
            "multi-thread": {
                "class": "logging.Formatter",
                "datefmt": "%I:%M:%S",
                "format": "%(levelname)-8s; %(threadName)s; %(name)-15s; %(pathname)s:%(funcName)s;%(lineno)d: %(message)s"
            },
            "verbose": {
                "class": "logging.Formatter",
                "datefmt": "%I:%M:%S",
                "format": "%(levelname)-8s; [%(process)d]; %(threadName)s; %(name)-15s; %(pathname)s:%(funcName)s;%(lineno)d: %(message)s"
            },
            "multiline": {
                "class": "logging.Formatter",
                "format": "Level: %(levelname)s\nTime: %(asctime)s\nProcess: %(process)d\nThread: %(threadName)s\nLogger: %(name)s\nPath: %(pathname)s:%(lineno)d\nFunction :%(funcName)s\nMessage: %(message)s\n"
            }
        },
        "handlers": {
            "console":{
                "level": "DEBUG",
                "class": "logging.StreamHandler",
                "formatter": "multiline",
                "stream" : "ext://sys.stdout"
            },
            "file_handler": {
                "level": "INFO",
                "class": "logging.handlers.WatchedFileHandler",
                "formatter": "verbose",
                "filename": "/tmp/file_handler.log",
                "mode": "a",
                "encoding": "utf-8"
            },
            "smtp": {
                "level": "ERROR",
                "class": "logging.handlers.SMTPHandler",
                "formatter": "multiline",
                "mailhost": ["127.0.0.1", 25],
                "fromaddr": "[email protected]",
                "toaddrs": ["[email protected]"],
                "subject": "Something went wrong"
            }
        },
        "loggers": { },
        "root": {
            "handlers": ["console"],
            "level": "DEBUG"
        }
    }
}

logging3.json

{
    "logging": {
        "version": 1,
        "disable_existing_loggers": true,
        "formatters": {
            "brief": {
                "class": "logging.Formatter",
                "style": "{",
                "datefmt": "%I:%M:%S",
                "format": "{levelname:8s}; {name:<15s}; {message:s}"
            },
            "simple": {
                "class": "logging.Formatter",
                "style": "{",
                "datefmt": "%I:%M:%S",
                "format": "{levelname:8s}; {asctime:s}; {name:<15s} {lineno:4d}; {message:s}"
            },
            "multi-process": {
                "class": "logging.Formatter",
                "style": "{",
                "datefmt": "%I:%M:%S",
                "format": "{levelname:8s}; {process:5d}; {asctime:s}; {name:<15s} {lineno:4d}; {message:s}"
            },
            "multi-thread": {
                "class": "logging.Formatter",
                "style": "{",
                "datefmt": "%I:%M:%S",
                "format": "{levelname:8s}; {threadName:5d}; {asctime:s}; {name:<15s} {lineno:4d}; {message:s}"
            },
            "verbose": {
                "class": "logging.Formatter",
                "style": "{",
                "datefmt": "%I:%M:%S",
                "format": "{levelname:8s}; {process:5d}; {threadName:8s}; {asctime:s}; {name:<15s} {lineno:4d}; {message:s}"
            },
            "multiline": {
                "class": "logging.Formatter",
                "style": "{",
                "datefmt": "%I:%M:%S",
                "format": "{levelname:8s}\n{process:5d}\n{threadName:8s}\n{asctime:s}\n{name:<15s}{lineno:4d}\n{message:s}\n"
            }
        },
        "handlers": {
            "console":{
                "level": "DEBUG",
                "class": "logging.StreamHandler",
                "formatter": "multiline",
                "stream" : "ext://sys.stdout"
            },
            "file_handler": {
                "level": "INFO",
                "class": "logging.handlers.WatchedFileHandler",
                "formatter": "verbose",
                "filename": "/tmp/file_handler.log",
                "mode": "a",
                "encoding": "utf-8"
            },
            "smtp": {
                "level": "ERROR",
                "class": "logging.handlers.SMTPHandler",
                "formatter": "multiline",
                "mailhost": ["127.0.0.1", 25],
                "fromaddr": "[email protected]",
                "toaddrs": ["[email protected]"],
                "subject": "Something went wrong"
            }
        },
        "loggers": { },
        "root": {
            "handlers": ["console"],
            "level": "DEBUG"
        }
    }
}

logging_example.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

""" An example of using JSON for log configuration """

from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import absolute_import


import json
import logging
import logging.config


def main():
    # each time we need to log something we can create a logger object
    # The operation of creating a logger should be quite cheap.
    # getLogger() without arguments returns the "root" logger.
    logger = logging.getLogger()
    logger.info("This is an INFO message on the root logger.")

    # If we need to separate things, we can always create child loggers:
    child = logging.getLogger().getChild("child")
    child.warning("This is a WARNING message on the child logger.")

    # let's create an error. This will send an email
    child.error("This is an ERROR message.")


if __name__ == "__main__":
    # create an initial logger. It will only log to console and it will disabled
    # when we read the logging configuration from the config file.
    # This logger can be useful when we need early logging. E.g. we may want to log
    # the location of the JSON file (e.g. if we get it from a CLI argument).
    logging.basicConfig(level="INFO")
    logger = logging.getLogger()
    logger.info("This is the logger configured by `logging.basicConfig()`.")

    # set up proper logging. This one disables the previously configured loggers.
    with open("logging.json", "r", encoding="utf-8") as fd:
        logging.config.dictConfig(json.load(fd))

    main()

python_logging_configuration.md

Example for configuring logging in Python 3 using a JSON file.

If you run example.py as it is, then you will get the following output:

INFO:root:This is the logger configured by `logging.basicConfig()`.
INFO    ; 2014-10-18 15:17:38,278; root           ; This is an INFO message on the root logger.
WARNING ; 2014-10-18 15:17:38,278; child          ; This is a WARNING message on the child logger.
ERROR   ; 2014-10-18 15:17:38,278; child          ; This is an ERROR message.
--- Logging error ---
Traceback (most recent call last):
  File "/usr/lib/python3.4/logging/handlers.py", line 971, in emit
    smtp = smtplib.SMTP(self.mailhost, port, timeout=self.timeout)
  File "/usr/lib/python3.4/smtplib.py", line 242, in __init__
    (code, msg) = self.connect(host, port)
  File "/usr/lib/python3.4/smtplib.py", line 321, in connect
    self.sock = self._get_socket(host, port, self.timeout)
  File "/usr/lib/python3.4/smtplib.py", line 292, in _get_socket
    self.source_address)
  File "/usr/lib/python3.4/socket.py", line 509, in create_connection
    raise err
  File "/usr/lib/python3.4/socket.py", line 500, in create_connection
    sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
Call stack:
  File "foo.py", line 44, in <module>
    main()
  File "foo.py", line 29, in main
    child.error("This is an ERROR message.")
Message: 'This is an ERROR message.'
Arguments: ()

This traceback is not a related to our application. It has to do with the logging configuration. More specifically we have set an smtp handler that tries to send an email using a MailServer that runs on 127.0.0.1:25, but there is no server listening to that ip/port. While developing, we can easily setup such a server by running the following command on a separate console session:

 sudo python -m smtpd -n -c DebuggingServer localhost:25 

Needless to say, on production we should set up a proper MailServer. Anyway, now, if we execute the script we will get the following output:

INFO:root:This is the logger configured by `logging.basicConfig()`.
INFO    ; 2014-10-18 15:17:51,078; root           ; This is an INFO message on the root logger.
WARNING ; 2014-10-18 15:17:51,079; child          ; This is a WARNING message on the child logger.
ERROR   ; 2014-10-18 15:17:51,079; child          ; This is an ERROR message.

while, on the console that runs the Mail Server we will get the following output:

---------- MESSAGE FOLLOWS ----------
From: [email protected]
To: [email protected]
Subject: Something went wrong
Date: Sat, 18 Oct 2014 12:17:51 -0000
X-Peer: 127.0.0.1

thread: MainThread
Level: ERROR
Time: 2014-10-18 15:17:51,079
Location: foo.py:29
Method: main
Message:
This is an ERROR message.
------------ END MESSAGE ------------