Logstash Test config

logstash-test.conf

# Where to get input
input {
  # Get input from syslog
  tcp {
    port => 5514
    type => "syslog"
  }
  udp {
    port => 5514
    type => "syslog"
  }
  # Get input from Lumberjack
  lumberjack {
    port => 5614
    type => "lumberjack"
    ssl_certificate => "/etc/logstash/ssl/logstash-forwarder.crt"
    ssl_key => "/etc/logstash/ssl/logstash-forwarder.key"
  }
  # Get input from CoreOS journal
  tcp {
    port  => 5714
    type  => "systemd"
    codec => json_lines
    tags  => ["coreos"]
  }
  # Get input as JSON lines
  tcp {
    port  => 5914
    type  => "json"
    codec => json_lines
    tags  => ["applogs"]
  }
  # Get application logs via log4j
  log4j {
    port  => 5814
    type  => "log4j"
    codec => "plain"
    tags  => ["applogs"]
  }
}

# Some Filtering
filter {
  # SYSLOG filter
  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    syslog_pri { }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }

    if !("_grokparsefailure" in [tags]) {
      mutate {
        replace => [ "message", "%{syslog_message}" ]
      }

      mutate {
        remove_field => [  "syslog_message" ]
      }
    }

    # Remove spurious fields that have names changed or been aggregated
    mutate {
      remove_field => [ "syslog_hostname", "syslog_timestamp" ]
    }
  }

  if [type] == "systemd" {
    mutate { rename => [ "MESSAGE", "message" ] }
    mutate { rename => [ "_SYSTEMD_UNIT", "program" ] }
  }

  # Docker filter
  if "docker/" in [program] {
    mutate {
      add_field => {
        "container_id" => "%{program}"
      }
    }
    mutate {
      gsub => [ "container_id", "docker/", "" ]
    }
    mutate {
      update => [ "program", "docker" ]
    }    
    mutate {
      rename => [ "log", "message" ]
    }
    date {
      match => [ "time", "ISO8601" ]
    }
  }
}

# Where to send output
output {
  # Send output to standard output device/interface
  stdout {
    codec => rubydebug
  }

  # Parse failed syslog messages
  if [type] == "syslog" and "_grokparsefailure" in [tags] {
    file { path => "/var/log/failed_syslog_events-%{+YYYY-MM-dd}" }
  }

  # Send output to Elasticsearch over HTTP interface.
  elasticsearch {
    protocol  => "http"
    cluster   => "ES_CLUSTER"
    host      => ["ES_HOST:ES_PORT"]
  }
}