Logstash

ls

ubuntu@ip-10-0-0-162:/etc/logstash$ ls -al
total 18856
drwxrwxr-x   3 root root     4096 Apr 17 20:03 .
drwxr-xr-x 115 root root     4096 Apr 17 14:00 ..
-rwxrwxrwx   1 root root 19296245 Apr 17 20:03 GeoLiteCity.dat
drwxrwxr-x   2 root root     4096 Apr 17 20:32 conf.d

nginx

NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:http_host} %{IPORHOST:client_ip} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent} "%{DATA:json_string}" %{BASE10NUM:request_duration}
NGINXERROR %{DATESTAMP:mydate} \[%{DATA:severity}\] (%{NUMBER:pid:int}#%{NUMBER}: \*%{NUMBER}|\*%{NUMBER}) %{GREEDYDATA:mymessage}

nginx.conf

input {
  file {
    type => "nginx_access"
    path => "/var/log/nginx/{,api.}{,development.,staging.}elephone.*access.log"
  }
  file {
    type => "nginx_error"
    path => "/var/log/nginx/{,api.}{,development.,staging.}elephone.*error.log"
  }
}

filter {
  if [type] == "nginx_access" {
    grok {
      match => { "message" => "%{NGINXACCESS}" }
    }
  }
  if [type] == "nginx_error" {
    grok {
      match => { "message" => "%{NGINXERROR}" }
    }
  }
  date {
    match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
  }
  geoip {
    source => "client_ip"
    target => "geoip"
    database => "/etc/logstash/GeoLiteCity.dat"
  }
  mutate {
    gsub => ["json_string", "\\x22", '"']
  }
  if [json_string] != "-" {
    json {
      source => "json_string"
    }
  }
}

output {
  elasticsearch {
    host => "localhost"
    protocol => "http"
  }
  stdout { codec => rubydebug }
}

stdout

{
             "message" => "api.development.elephone.com xx.xx.xx.xx [17/Apr/2015:20:33:31 +0000] \"GET /zf-apigility-welcome/img/ag-hero.png HTTP/1.1\" 404 559 \"http://api.development.elephone.com/\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/41.0.2272.76 Chrome/41.0.2272.76 Safari/537.36\" \"-\" 0.255 0.255",
            "@version" => "1",
          "@timestamp" => "2015-04-17T20:33:31.000Z",
                "type" => "nginx_access",
                "host" => "ip-10-0-0-162",
                "path" => "/var/log/nginx/api.development.elephone.com-access.log",
           "http_host" => "api.development.elephone.com",
           "client_ip" => "xx.xx.xx.xx",
           "timestamp" => "17/Apr/2015:20:33:31 +0000",
                "verb" => "GET",
             "request" => "/zf-apigility-welcome/img/ag-hero.png",
         "httpversion" => "1.1",
            "response" => "404",
               "bytes" => "559",
            "referrer" => "http://api.development.elephone.com/",
               "agent" => "\"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/41.0.2272.76 Chrome/41.0.2272.76 Safari/537.36\"",
         "json_string" => "-",
    "request_duration" => 0.255
}