Convert hypernode nginx json logging to apache combined log format

convert-log-to-combined.py

#!/usr/bin/env python
""" Minimal script to convert json access logging on nginx to combined log format.
    To use, save the file as convert.py and make it executable: chmod +x convert.py
    
    Then you can convert the files using cat:
        cat access.log | ./convert.py       
        
    This will send the log output to stdout, and a lines ignore count to sderr.
"""
import sys
import json
import datetime


def print_report(ignored, total):
    print >>sys.stderr, "#" * 50
    print >>sys.stderr, "# lines ignored: {} on a total of {} lines".format(ignored, total)
    print >>sys.stderr, "#" * 50


def format_time(time):
    # IN: 2016-12-22T10:47:28+00:00
    # OUT: [01/Jul/2002:12:11:52 +0000]
    timetuple = datetime.datetime.strptime(time, "%Y-%m-%dT%H:%M:%S+00:00")
    return datetime.datetime.strftime(timetuple, "[%d/%b/%Y:%H:%M:%S +0000]")


def format_access_log(logdict):
    return '{host} - {user} {time} "{request}" {status} {bytes} "{referer}" "{user_agent}"'.format(
        host=logdict.get('remote_addr', '-'),
        user=logdict.get('remote_user', '-'),
        time=format_time(logdict.get('time')),
        request=logdict.get('request', '-'),
        status=logdict.get('status', '-'),
        bytes=logdict.get('body_bytes_sent', '-'),
        referer=logdict.get('referer', '-'),
        user_agent=logdict.get('user_agent', '-'),
    )


total_lines = 0
lines_ignored = 0

for line in sys.stdin.readlines():
    total_lines += 1
    logline = line.strip()
    try:
        logdict = json.loads(logline)
    except ValueError:
      lines_ignored += 1
    print >>sys.stdout, format_access_log(logdict)

print_report(ignored=lines_ignored, total=total_lines)