Default nginx log format (combined) and grok pattern

README.md

Pattern file: /etc/logstash/patterns/nginx Logstash shipper: /etc/logstash/conf.d/shipper.conf

Nginx http log module: http://nginx.org/en/docs/http/ngx_http_log_module.html

nginx-default-log-format

log_format combined '$remote_addr - $remote_user [$time_local] '
                    '"$request" $status $body_bytes_sent '
                    '"$http_referer" "$http_user_agent"';

nginx.pattern

NGINX_ACCESS %{IPORHOST:remote_addr} - %{USERNAME:remote_user} \[%{HTTPDATE:time_local}\] \"%{DATA:request}\" %{INT:status} %{NUMBER:bytes_sent} \"%{DATA:http_referer}\" \"%{DATA:http_user_agent}\"

shipper.conf

input {
  file {
    type => "nginx"
    start_position => "beginning"
    path => [ "/var/log/nginx/*.log" ]
  }
}

filter {
  if [type] == "nginx" {
    grok {
      patterns_dir => "/etc/logstash/patterns"
      match => { "message" => "%{NGINX_ACCESS}" }
      remove_tag => [ "_grokparsefailure" ]
      add_tag => [ "nginx_access" ]
    }

    geoip {
      source => "remote_addr"
    }
  }
}

output {
  redis {
    host => "<your redis host>"
    data_type => "list"
    key => "logstash"
    codec => json
  }
}