Created
May 21, 2015 15:47
-
-
Save jhmartin/c09e450853e273ee88f2 to your computer and use it in GitHub Desktop.
Multiple Elasticsearch outputs in logstash
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#http://pastebin.com/xGt6vv9R from yardenbar | |
output { | |
if [type] == "postfix" { | |
elasticsearch { | |
host => [ "ES_HOSTNAME" ] | |
protocol => "transport" | |
cluster => "elasticsearch" | |
index => "postfix-%{+YYYY.MM.dd}" | |
manage_template => true | |
template_overwrite => true | |
template => "/etc/logstash/templates.d/postfix.json" | |
template_name => "postfix" | |
} | |
} | |
else if [type] in ["nginx", "nginx_error"] { | |
elasticsearch { | |
host => [ "ES_HOSTNAME" ] | |
protocol => "transport" | |
cluster => "elasticsearch" | |
index => "nginx-%{+YYYY.MM.dd}" | |
manage_template => true | |
template_overwrite => true | |
template => "/etc/logstash/templates.d/nginx.json" | |
template_name => "nginx" | |
} | |
} | |
else if [type] == "rails" { | |
elasticsearch { | |
host => [ "ES_HOSTNAME" ] | |
protocol => "transport" | |
cluster => "elasticsearch" | |
index => "rails-%{+YYYY.MM.dd}" | |
manage_template => true | |
template_overwrite => true | |
template => "/etc/logstash/templates.d/rails.json" | |
template_name => "rails" | |
} | |
} | |
else if [type] == "webpagetest" { | |
elasticsearch { | |
host => [ "ES_HOSTNAME" ] | |
protocol => "transport" | |
cluster => "elasticsearch" | |
index => "webpagetest-global" | |
manage_template => true | |
template_overwrite => true | |
template => "/etc/logstash/templates.d/webpagetest.json" | |
template_name => "webpagetest" | |
} | |
} | |
else if [type] == "push-notification" { | |
elasticsearch { | |
host => [ "ES_HOSTNAME" ] | |
protocol => "transport" | |
cluster => "elasticsearch" | |
index => "push-notification-%{+YYYY.MM.dd}" | |
manage_template => true | |
template_overwrite => true | |
template => "/etc/logstash/templates.d/push_notification.json" | |
template_name => "push_notification" | |
} | |
} | |
else if [type] == "fastly" { | |
elasticsearch { | |
host => [ "ES_HOSTNAME" ] | |
protocol => "transport" | |
cluster => "elasticsearch" | |
index => "fastly-%{+YYYY.MM.dd}" | |
manage_template => true | |
template_overwrite => true | |
template => "/etc/logstash/templates.d/fastly.json" | |
template_name => "fastly" | |
} | |
} | |
else { | |
file { | |
path => "/var/log/logstash/unknown_messages.log" | |
} | |
} | |
} |
Great, I'm searching it for a longtime
If you don't want to have the type fields in the documents that are output, you can also consider adding metadata which will be used for driving data to different outputs, and removing the 'type' fields. I have written a blog post that demonstrates this approach at https://alexmarquardt.com/2018/08/31/using-logstash-to-drive-filtered-data-from-a-single-source-into-multiple-output-destinations/.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
@jhmartin can you please share the json templates please. it will be a treat for us. Thank you very much for this.