Forked : Python nginx Log Parser <<Modified>> : Parsed & normalized the datetime string to native python objects

parser.py

#!/usr/bin/env python
import gzip
import os
import sys
import re
from datetime import datetime
import pytz

tz = pytz.timezone('UTC')

INPUT_DIR = "nginx-logs"

lineformat = re.compile(r"""(?P<ipaddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - \[(?P<dateandtime>\d{2}\/[a-z]{3}\/\d{4}:\d{2}:\d{2}:\d{2} (\+|\-)\d{4})\] ((\"(GET|POST) )(?P<url>.+)(http\/1\.1")) (?P<statuscode>\d{3}) (?P<bytessent>\d+) (["](?P<refferer>(\-)|(.+))["]) (["](?P<useragent>.+)["])""", re.IGNORECASE)

for f in os.listdir(INPUT_DIR):
    if f.endswith(".gz"):
        logfile = gzip.open(os.path.join(INPUT_DIR, f))
    else:
        logfile = open(os.path.join(INPUT_DIR, f))

    for l in logfile.readlines():
        data = re.search(lineformat, l)
        if data:
            datadict = data.groupdict()
            ip = datadict["ipaddress"]
            datetimeobj = datetime.strptime(datadict["dateandtime"], "%d/%b/%Y:%H:%M:%S %z") # Converting string to datetime obj
            url = datadict["url"]
            bytessent = datadict["bytessent"]
            referrer = datadict["refferer"]
            useragent = datadict["useragent"]
            status = datadict["statuscode"]
            method = data.group(6)

            print ip, \
              tz.normalize(datetimeobj), \
              url, \
              bytessent, \
              referrer, \
              useragent, \
              status, \
              method

    logfile.close()