-
-
Save nmische/4261613 to your computer and use it in GitHub Desktop.
| input { | |
| tcp { | |
| type => "iis_advanced_full" | |
| port => 3333 | |
| } | |
| } | |
| filter { | |
| grok { | |
| type => "iis_advanced_full" | |
| pattern => "(?:-|\"%{IP:x_forwarded_for}\") %{NUMBER:sc_win32_status} (?:-|%{NONNEGINT:w3wp-privatebytes}) (?:-|%{DATA:username}) (?:-|\"%{DATA:agent}\") %{URIPATHPARAM:request} (?:-|%{DATA:querysting}) %{TIME:time} %{TIME:time_local} %{NUMBER:time_taken_ms} %{INT:sc_substatus} %{INT:status} (?:-|\"%{IPORHOST:s_sitename}\") %{IP:s_ip} %{POSINT:s_port} \"%{DATA:s_computername}\" (?:-|%{NUMBER:requestspersecond}) (?:-|\"%{URI:cs_referrer}\") (?:-|\"%{DATA:s_proxy}\") (?:-|\"%{DATA:cs_version}\") (?:-|\"%{DATA:c_protocol}\") (?:-|%{WORD:cs_method}) (?:-|\"%{IPORHOST:cs_host}\") %{TIMESTAMP_ISO8601:endrequest_utc} %{DATE_EU:date} %{DATE_EU:date_local} (?:-|%{NUMBER:cpu_utilization}) (?:-|\"%{DATA:cs_cookie}\") (?:-|\"%{DATA:s_contentpath}\") %{IP:c_ip} %{NUMBER:sc_bytes} %{NUMBER:cs_bytes} %{TIMESTAMP_ISO8601:timestamp}" | |
| } | |
| mutate { | |
| gsub => [ | |
| "timestamp", "(.*) (.*)", "\1 \2 +0000" | |
| ] | |
| } | |
| date { | |
| type => "iis_advanced_full" | |
| # Try to pull the timestamp from the 'timestamp' field (parsed above with | |
| # grok). The apache time format looks like: "18/Aug/2011:05:44:34 -0700" | |
| timestamp => "yyyy-MM-dd kk:mm:ss.SSS Z" | |
| } | |
| } | |
| output { | |
| # Use stdout in debug mode again to see what logstash makes of the event. | |
| stdout { | |
| debug => true | |
| debug_format => "json" | |
| } | |
| elasticsearch { | |
| # Setting 'embedded' will run a real elasticsearch server inside logstash. | |
| # This option below saves you from having to run a separate process just | |
| # for ElasticSearch, so you can get started quicker! | |
| embedded => true | |
| } | |
| } |
If you're showing up and reading this on your hunt to build a working parser:
use this site:
https://grokdebug.herokuapp.com/
look at the pattern of your logs and map this format to the patterns shown above. Mine looked like this:
%{TIMESTAMP_ISO8601:log_timestamp} %{TIMESTAMP_ISO8601:local_timestamp} (?:-|%{NUMBER:requestspersecond}) (?:-|%{DATA:cs_uri_stem}) %{IP:s_ip} %{INT:status} %{NUMBER:sc_substatus} %{NUMBER:sc_bytes} (?:-|%{WORD:cs_method}) (?:-|%{DATA:querystring}) %{IP:c_ip} (?:-|"%{DATA:c_protocol}") (?:-|"%{DATA:s_proxy}") (?:-|%{DATA:username}) (?:-|"%{DATA:s_contentpath}") (?:-|"%{IPORHOST:cs_host}") %{TIMESTAMP_ISO8601:beginrequest_utc} %{TIMESTAMP_ISO8601:endrequest_utc} %{NUMBER:time_taken_ms} (?:-|"%{IPORHOST:s_sitename}") (?:-|"%{DATA:agent}") (?:-|"%{URI:cs_referrer}") %{NUMBER:sc_win32_status} (?:-|%{NONNEGINT:w3wp-privatebytes}) (?:-|"%{DATA:cs_version}") (?:-|"%{DATA:cs_cookie}") "%{DATA:s_computername}" %{POSINT:s_port} (?:-|"%{IP:x_forwarded_for}")
Thank you!