Created
December 11, 2012 20:00
-
-
Save nmische/4261613 to your computer and use it in GitHub Desktop.
Logstash Config for IIS Advanced Logging
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
input { | |
tcp { | |
type => "iis_advanced_full" | |
port => 3333 | |
} | |
} | |
filter { | |
grok { | |
type => "iis_advanced_full" | |
pattern => "(?:-|\"%{IP:x_forwarded_for}\") %{NUMBER:sc_win32_status} (?:-|%{NONNEGINT:w3wp-privatebytes}) (?:-|%{DATA:username}) (?:-|\"%{DATA:agent}\") %{URIPATHPARAM:request} (?:-|%{DATA:querysting}) %{TIME:time} %{TIME:time_local} %{NUMBER:time_taken_ms} %{INT:sc_substatus} %{INT:status} (?:-|\"%{IPORHOST:s_sitename}\") %{IP:s_ip} %{POSINT:s_port} \"%{DATA:s_computername}\" (?:-|%{NUMBER:requestspersecond}) (?:-|\"%{URI:cs_referrer}\") (?:-|\"%{DATA:s_proxy}\") (?:-|\"%{DATA:cs_version}\") (?:-|\"%{DATA:c_protocol}\") (?:-|%{WORD:cs_method}) (?:-|\"%{IPORHOST:cs_host}\") %{TIMESTAMP_ISO8601:endrequest_utc} %{DATE_EU:date} %{DATE_EU:date_local} (?:-|%{NUMBER:cpu_utilization}) (?:-|\"%{DATA:cs_cookie}\") (?:-|\"%{DATA:s_contentpath}\") %{IP:c_ip} %{NUMBER:sc_bytes} %{NUMBER:cs_bytes} %{TIMESTAMP_ISO8601:timestamp}" | |
} | |
mutate { | |
gsub => [ | |
"timestamp", "(.*) (.*)", "\1 \2 +0000" | |
] | |
} | |
date { | |
type => "iis_advanced_full" | |
# Try to pull the timestamp from the 'timestamp' field (parsed above with | |
# grok). The apache time format looks like: "18/Aug/2011:05:44:34 -0700" | |
timestamp => "yyyy-MM-dd kk:mm:ss.SSS Z" | |
} | |
} | |
output { | |
# Use stdout in debug mode again to see what logstash makes of the event. | |
stdout { | |
debug => true | |
debug_format => "json" | |
} | |
elasticsearch { | |
# Setting 'embedded' will run a real elasticsearch server inside logstash. | |
# This option below saves you from having to run a separate process just | |
# for ElasticSearch, so you can get started quicker! | |
embedded => true | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
If you're showing up and reading this on your hunt to build a working parser:
use this site:
https://grokdebug.herokuapp.com/
look at the pattern of your logs and map this format to the patterns shown above. Mine looked like this:
%{TIMESTAMP_ISO8601:log_timestamp} %{TIMESTAMP_ISO8601:local_timestamp} (?:-|%{NUMBER:requestspersecond}) (?:-|%{DATA:cs_uri_stem}) %{IP:s_ip} %{INT:status} %{NUMBER:sc_substatus} %{NUMBER:sc_bytes} (?:-|%{WORD:cs_method}) (?:-|%{DATA:querystring}) %{IP:c_ip} (?:-|"%{DATA:c_protocol}") (?:-|"%{DATA:s_proxy}") (?:-|%{DATA:username}) (?:-|"%{DATA:s_contentpath}") (?:-|"%{IPORHOST:cs_host}") %{TIMESTAMP_ISO8601:beginrequest_utc} %{TIMESTAMP_ISO8601:endrequest_utc} %{NUMBER:time_taken_ms} (?:-|"%{IPORHOST:s_sitename}") (?:-|"%{DATA:agent}") (?:-|"%{URI:cs_referrer}") %{NUMBER:sc_win32_status} (?:-|%{NONNEGINT:w3wp-privatebytes}) (?:-|"%{DATA:cs_version}") (?:-|"%{DATA:cs_cookie}") "%{DATA:s_computername}" %{POSINT:s_port} (?:-|"%{IP:x_forwarded_for}")