Quick'n dirty HTTP combined access log JSON formatter

commonlog2json.go

package main

import (
    "bufio"
    "fmt"
    "os"
    "regexp"
)

func main() {
    scanner := bufio.NewScanner(os.Stdin)
    lineJson := []byte{}

    re := regexp.MustCompile(`^(?P<remote_addr>[0-9a-f.:]+)\s` +
        `(?P<clientid>[\w.-]+)\s` +
        `(?P<remote_user>[\w.-]+)\s` +
        `\[(?P<date>[^\[\]]+)\]\s` +
        `"(?:(?P<method>[A-Z]+)\s` +
        `(?P<request_uri>.+)\s` +
        `(?P<protocol>HTTP/[0-9.]+)|-)"\s` +
        `(?P<status>\d{3})\s` +
        `(?P<size>\d+|-)\s` +
        `"(?P<referrer>(?:[^"]|\")+)"\s` +
        `"(?P<ua>(?:[^"]|\")*)"$`)

    for scanner.Scan() {
        line := scanner.Bytes()
        fmt.Println(string(re.Expand(lineJson, []byte(`{`+
            `"request_method":"${method}",`+
            `"request_uri":"${request_uri}",`+
            `"server_protocol":"${protocol}",`+
            `"remote_addr":"${remoteaddr}",`+
            `"remote_user":"${userid}",`+
            `"date":"${date}",`+
            `"status":${status},`+
            `"body_bytes_sent":${size}`+
            `}`), line, re.FindSubmatchIndex(line))))
    }
}

Usage: pipe your HTTP access logs in "combined" format to it and it will output the JSON-formatted version to stdout:

$ tail /var/log/nginx/access.log
74.59.x.y - - [02/Nov/2014:15:36:57 +0100] "GET /fonts/ubuntu-medium.woff HTTP/1.1" 200 24616 "http://falzon.me/style.css" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36"
74.59.x.y - - [02/Nov/2014:15:36:57 +0100] "GET /img/favicon.png HTTP/1.1" 200 4071 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36"
74.59.x.y - - [02/Nov/2014:15:37:29 +0100] "GET /stuff/index.html HTTP/1.1" 200 3976 "http://falzon.me/" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36"
202.46.x.y - - [02/Nov/2014:15:48:55 +0100] "GET / HTTP/1.1" 200 3301 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)"

$ tail /var/log/nginx/access.log | go run httplogparse.go
{"request_method":"GET","request_uri":"/fonts/ubuntu-medium.woff","server_protocol":"HTTP/1.1","remote_addr":"74.59.x.y","remote_user":"-","date":"02/Nov/2014:15:36:57 +0100","status":200,"body_bytes_sent":24616}
{"request_method":"GET","request_uri":"/img/favicon.png","server_protocol":"HTTP/1.1","remote_addr":"74.59.x.y","remote_user":"-","date":"02/Nov/2014:15:36:57 +0100","status":200,"body_bytes_sent":4071}
{"request_method":"GET","request_uri":"/stuff/index.html","server_protocol":"HTTP/1.1","remote_addr":"74.59.x.y","remote_user":"-","date":"02/Nov/2014:15:37:29 +0100","status":200,"body_bytes_sent":3976}
{"request_method":"GET","request_uri":"/","server_protocol":"HTTP/1.1","remote_addr":"202.46.x.y","remote_user":"-","date":"02/Nov/2014:15:48:55 +0100","status":200,"body_bytes_sent":3301}

Feel free to hack the format of the output JSON, mine was pretty much tailored to my needs.