logstash grok filter for Elasticsearch logs

logstash.conf

filter {
  if [type] == "elasticsearch" {
    grok {
      match => [ "message", "\[%{TIMESTAMP_ISO8601:timestamp}\]\[%{DATA:severity}%{SPACE}\]\[%{DATA:source}%{SPACE}\]%{SPACE}(?<message>(.|\r|\n)*)" ]
      overwrite => [ "message" ]
    }

    if "_grokparsefailure" not in [tags] {
      grok {  # regular logs
        match => [
          "message", "^\[%{DATA:node}\] %{SPACE}\[%{DATA:index}\]%{SPACE}(?<short_message>(.|\r|\n)*)",
          "message", "^\[%{DATA:node}\]%{SPACE}(?<short_message>(.|\r|\n)*)" ]
        tag_on_failure => []
      }

      grok {  # slow logs
        match => [ "message", "took\[%{DATA:took}\], took_millis\[%{NUMBER:took_millis}\], types\[%{DATA:types}\], stats\[%{DATA:stats}\], search_type\[%{DATA:search_type}\], total_shards\[%{NUMBER:total_shards}\], source\[%{DATA:source_query}\], extra_source\[%{DATA:extra_source}\]," ]
        tag_on_failure => []
        add_tag => [ "elasticsearch-slowlog" ]
      }

      date { # use timestamp from the log
        "match" => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS" ]
        target => "@timestamp"
      }

      mutate {
        remove_field => [ "timestamp" ]  # remove unused stuff
      }
    }
  }
}

Thanks for this Gist!! I made a couple of minor changes to cast the "took_millis" and "total_shards" as INTs in the Grok pattern, and also added the "shard" number which appears after the "index" field. This Gist was very helpful!

Thanks, @CliveJL! This is where it originated, by the way: https://sematext.com/blog/2015/01/19/grok-elasticsearch-logs-with-logstash But it's version specific. This specific one was tested on 2.2

This Gist is very helpful!
Thank you very much indeed!

I've added a new log parse for mapping failures, as we are moving from 1.7 to 5.5 and hunting them down currently.

grok { # Mapping failures
  match => [
  "short_message", "(%{SPACE}|\n)java.lang.IllegalArgumentException: \[%{DATA:failing_field}\] is defined as (a|an) %{WORD:existing_mapping} in mapping \[%{WORD:target_type}\] but this name is already used for (an|a) %{WORD:other_type} in other types(?<stacktrace>(.|\r|\n)*)"
  ]
  tag_on_failure => []
  add_tag => [ "elasticsearch-mappingfailure" ]
 }