Logstash config to structure "syslog { docker { kafka | zk | go | x } | system }" logs

logstash.conf.sh

input {
  tcp {
    port => 6514
  }
}

filter {
    # Match docker services
    grok {
      break_on_match => true
      match => {
        "message" => "%{SYSLOGTIMESTAMP} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:source} (?<service>[a-zA-Z0-9\-\._]+)?\/%{WORD:containerid}\[%{INT}\]:"
      }
      overwrite => [ "service" ]
      add_tag => ["docker"]
    }
    # Match golang logs formatted using logrus (k=v)
    grok {
      break_on_match => true
      match => {
        "message" => [
          "\"%{TIMESTAMP_ISO8601}\" level=%{LOGLEVEL:level} msg=\"%{DATA:message}\" %{GREEDYDATA:gokv}",
          "\"%{TIMESTAMP_ISO8601}\" level=%{LOGLEVEL:level} msg=%{DATA:message} %{GREEDYDATA:gokv}"
        ]
      }
      overwrite => [ "message" ]
      add_tag => ["go"]
    }
    # Extract key/values
    kv {
      source => "gokv"
      remove_field => [ "gokv" ]
    }
    # Match Kafka logs
    grok {
      break_on_match => true
      match => {
        "message" => [
          "\[%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}\]\s%{LOGLEVEL:level}\s%{GREEDYDATA:message}$"
        ]
      }
      add_tag => ["kafka"]
      overwrite => [ "message" ]
    }
    # Match Zookeeper logs
    grok {
      break_on_match => true
      match => {
        "message" => [
          "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME} \[myid:%{INT:zkid}\]\s-\s%{LOGLEVEL:level}\s*\[%{GREEDYDATA:class}\]\s-\s%{GREEDYDATA:message}$"
        ]
      }
      add_tag => ["zk"]
      overwrite => [ "message" ]
    }
    # Match Java stacktrace
    grok {
      break_on_match => true
      match => {
        "message" => [
          "\s#011at\s%{GREEDYDATA:message}$"
        ]
      }
      add_tag => ["stacktrace"]
      overwrite => [ "message" ]
    }
    # In the end match syslog logs
    grok {
      break_on_match => true
      match => {
        "message" => [
          "%{SYSLOGBASE} %{GREEDYDATA:message}"
        ]
      }
      add_tag => ["syslog"]
      overwrite => [ "message"]
    }
    # Match iptables logs
    grok {
      break_on_match => true
      match => {
        "message" => [
          "\[DROPPED by iptables\]%{GREEDYDATA:message}"
        ]
      }
      add_tag => ["iptables"]
      overwrite => [ "message" ]
    }
    # Clean up
    mutate {
      remove_field => ["@version", "pid", "host", "port"]
      remove_tag => ["_grokparsefailure"]
      rename => ["logsource", "source"]
      #rename => ["program", "service"]
      lowercase => ["level"]
    }
    if ![level] {
      mutate {
        remove_field => ["level"]
      }
    }
    if "iptables" in [tags] {
      mutate {
        remove_field => ["program"]
        add_field => { "service" => "iptables" }
        add_field => { "level" => "warn" }
      }
    }
}

output {
  stdout { codec => json }
}