Last active
March 15, 2016 16:49
-
-
Save toast38coza/9eea9786c3394c5ff22d to your computer and use it in GitHub Desktop.
Logstash config for parsing incoming nginx logs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| input { | |
| stdin { } | |
| beats { | |
| port => 5044 | |
| type => nginx | |
| } | |
| } | |
| filter { | |
| grok { | |
| match => { "message" => "%{NGINXACCESS}"} | |
| } | |
| geoip { | |
| source => "clientip" | |
| } | |
| } | |
| ## Add your filters here | |
| output { | |
| elasticsearch { | |
| hosts => "192.168.99.100:9200" | |
| index => "%{index}-%{type}-%{+YYYY.MM.dd}" | |
| } | |
| stdout { codec => rubydebug } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| NGUSERNAME [a-zA-Z\.\@\-\+_%]+ | |
| NGUSER %{NGUSERNAME} | |
| NGINXACCESS %{IPORHOST:clientip} %{NGUSER:ident} %{NGUSER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 1.2.3.4 - joe [14/Mar/2016:07:48:05 +0000] "GET /some/page/ HTTP/1.1" 200 6426 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| $ logstash -f logstash/examples/nginx.conf | |
| Settings: Default pipeline workers: 4 | |
| Logstash startup completed | |
| 1.2.3.4 - joe [14/Mar/2016:07:48:05 +0000] "GET /some/page/ HTTP/1.1" 200 6426 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36" | |
| { | |
| "message" => "1.2.3.4 - joe [14/Mar/2016:07:48:05 +0000] \"GET /some/page/ HTTP/1.1\" 200 6426 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36\"", | |
| "@version" => "1", | |
| "@timestamp" => "2016-03-15T15:55:06.848Z", | |
| "host" => "Christos-MacBook-Pro.local", | |
| "clientip" => "1.2.3.4", | |
| "ident" => "-", | |
| "auth" => "joe", | |
| "timestamp" => "14/Mar/2016:07:48:05 +0000", | |
| "verb" => "GET", | |
| "request" => "/some/page/", | |
| "httpversion" => "1.1", | |
| "response" => "200", | |
| "bytes" => "6426", | |
| "agent" => "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36\"" | |
| "geoip" => { | |
| "ip" => "132.248.44.99", | |
| "country_code2" => "MX", | |
| "country_code3" => "MEX", | |
| "country_name" => "Mexico", | |
| "continent_code" => "NA", | |
| "region_name" => "09", | |
| "city_name" => "Mexico", | |
| "latitude" => 19.434200000000004, | |
| "longitude" => -99.1386, | |
| "timezone" => "America/Mexico_City", | |
| "real_region_name" => "Distrito Federal", | |
| "location" => [ | |
| [0] -99.1386, | |
| [1] 19.434200000000004 | |
| ] | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment