Skip to content

Instantly share code, notes, and snippets.

@ei-grad
Last active November 24, 2017 04:37
Show Gist options
  • Save ei-grad/51098fc1dce9490bf62f to your computer and use it in GitHub Desktop.
Save ei-grad/51098fc1dce9490bf62f to your computer and use it in GitHub Desktop.
Logstash example config to parse apache combined log
input { stdin { type => "access" } }
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}" }
remove_field => [ "message" ]
}
date {
match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
remove_field => [ "timestamp" ]
}
geoip {
source => "clientip"
}
mutate {
convert => {
"bytes" => "integer"
"response" => "integer"
}
}
}
output { elasticsearch {
hosts => ["elasticsearch"]
template => "/template.json"
template_overwrite => true
} }
cat $* | docker run -i --rm \
-v "$PWD/template.json:/template.json" \
-v "$PWD/logstash-apache.conf:/logstash.conf" \
--link elasticsearch
logstash logstash --config /logstash.conf
{
"template": "logstash-*",
"settings": {"index.refresh_interval": "-1"},
"mappings": {
"_default_": {
"_all": {"enabled": false},
"date_detection": false,
"dynamic_templates": [
{"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {"type": "keyword"}
}}
],
"properties": {
"@timestamp": {"type": "date", "format": "dateOptionalTime"},
"agent": {"type": "text", "fields": {"raw": {"type": "keyword"}}},
"referrer": {"type": "text", "fields": {"raw": {"type": "keyword"}}},
"request": {"type": "text", "fields": {"raw": {"type": "keyword"}}},
"host": {"type": "keyword"},
"httpversion": {"type": "keyword"},
"bytes": {"type": "long"},
"response": {"type": "short"},
"clientip": {"type": "ip"},
"geoip": {
"type": "object", "dynamic": true,
"properties": {"location": {"type": "geo_point"}}
}
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment