Logstash configuration for Mesos

mesos.conf

# Logstash Configuration for Mesos Masters and Slaves
input {
  file { 
    path => "/var/log/mesos/mesos-master.[EIFW]*"
    tags => [
      "master"
    ]
  }

  file { 
    path => "/var/log/mesos/mesos-slave.[EIFW]*"
    tags => [
      "slave"
    ]
  }
 
  file {
    path => "/tmp/mesos/slaves/*/frameworks/*/executors/*/runs/latest/std*"
    tags => [
      "slave"
    ]
  }
}

filter {
  grok {
    match => {
      "message" => "(?<log_severity>[EFIW])%{MONTHNUM:month}%{MONTHDAY:day}\s+%{TIME:time}\s+%{BASE10NUM:thread_id}\s+%{NOTSPACE:file}\:%{BASE10NUM:line}\]\s+%{GREEDYDATA:log_data}"
    }
    add_field => {
      "date_time" => "2015-%{month}-%{day}T%{time}Z"
    }
  }

  grok {
    match => {
      "path" => "/%{GREEDYDATA}/executors/%{GREEDYDATA:app}[.]%{GREEDYDATA}"
    }
    add_field => {
      "date_time" => "%{@timestamp}"
    }
  }

  if [log_severity] =~ "E" {
    mutate {
      replace => {
        "log_severity" => "ERROR"
      }
    }
  }

  if [log_severity] =~ "F" {
    mutate {
      replace => {
        "log_severity" => "FATAL"
      }
    }
  }

  if [log_severity] =~ "I" {
    mutate {
      replace => {
        "log_severity" => "INFO"
      }
    }
  }

  if [log_severity] =~ "W" {
    mutate {
      replace => {
        "log_severity" => "WARNING"
      }
    }
  }

  mutate {
    convert => {
      "day" => "integer"
    }
  }

  mutate {
    convert => {
      "month" => "integer"
    }
  }

  mutate {
    convert => {
      "line" => "integer"
    }
  }

  mutate {
    convert => {
      "thread_id" => "integer"
    }
  }

  date {
    match => [
      "date_time", "ISO8601"
    ]
  }
}

output {
  elasticsearch {
    "codec" => plain
    "host" => [
      "search-logstash-production-2agp3zffh45exrxsdddpvf4g3u.us-west-2.es.amazonaws.com"
    ]
    "index" => "mesos-production-%{+YYYYMMdd}"
    "protocol" => "http"
    "port" => "443"
    "ssl" => "true"
  }
}