Logstash stuff for MapR

README.md

On the client nodes (Redhat):

yum -y install https://download.elastic.co/logstash-forwarder/binaries/logstash-forwarder-0.4.0-1.x86_64.rpm

Ubuntu:

wget https://download.elastic.co/logstash-forwarder/binaries/logstash-forwarder_0.4.0_amd64.deb
dpkg -i logstash-forwarder_0.4.0_amd64.deb

Copy the SSL .crt from the logstash server to the node where you just installed the forwarder:

# scp /etc/logstash/lumberjack.crt node2:/opt/logstash-forwarder/lumberjack.crt

Start it up

service logstash-forwarder start

logstash-forwarder.conf

{
  "network": {
    "servers": [ "utility:5043" ],
    "timeout": 15,
#    "ssl certificate": "./logstash-forwarder.crt",
#    "ssl key": "./logstash-forwarder.key",
    "ssl ca": "/opt/logstash-forwarder/lumberjack.crt"
  },
  "files": [
    {
      "paths": [
        "/var/log/messages",
        "/opt/mapr/logs/*"
      ]
    }
  ]
}

lumberjack.conf

input {
  lumberjack {
    # The port to listen on
    port => 5043

    # The paths to your ssl cert and key
    ssl_certificate => "/etc/logstash/lumberjack.crt"
    ssl_key => "/etc/logstash/lumberjack.key"

    # Set this to whatever you want.
    type => "syslog"
  }
}

filter {
  grok {
# "2015-03-16 16:35:17,483 INFO ZKDataRetrieval [Thread-2-EventThread]: Process path: /services_config/cldb/node2. Event state: SyncConnected. Event type: NodeDataChanged",
# "2015-03-16 16:41:30,217 INFO  com.mapr.warden.centralconfig.PullCentralConfigTaskScheduler [PullCentralConfigTask]: /opt/mapr/server/pullcentralconfig process terminated with status: 0"
# 2015-03-17 11:36:02:INFO:7312: maprexecute renice by uid 2147483632 gid 2147483632
   match => { "message" => "%{DATESTAMP:datestamp}:%{LOGLEVEL:loglevel}:%{NUMBER}:\s+%{GREEDYDATA:message}" }
   match => { "message" => "%{DATESTAMP:datestamp}\s+%{LOGLEVEL:loglevel}\s+%{DATA:program}\s%{DATA:thread}:\s+%{GREEDYDATA:message}" }
   match => { "message" => "%{DATESTAMP:datestamp}\s+%{LOGLEVEL}\s+%{DATA:class}:%{NUMBER:line}\s+%{GREEDYDATA:message}" }
   overwrite => [ "message" ]
  }
}

output {
  elasticsearch {
    host => "localhost"
    index => "lumberjack"
  }
  stdout { codec => rubydebug }
}