Logstash configuration

01-inputs.conf

#logstash-forwarder
input {
  lumberjack {
    port => 5043
    type => "logs"
    ssl_certificate => "/usr/local/etc/ssl/metrics.int.unixathome.org.crt"
    ssl_key         => "/usr/local/etc/ssl/metrics.int.unixathome.org.nopassword.key"
    codec     => plain { charset => "ISO-8859-1" }
  }
}

#tcp syslog stream via 5140
input {
  tcp {
    type => "syslog"
    port => 5140
  }
}
#udp syslogs tream via 5140
input {
  udp {
    type => "syslog"
    port => 5140
  }
}

10-syslog.conf

filter {
  if [type] == "syslog" {

    #change to pfSense ip address
    if [host] =~ /10\.55\.0\.1/ {
      mutate {
        add_tag => ["PFSense", "Ready"]
      }
    }

    if "Ready" not in [tags] {
      mutate {
        add_tag => [ "syslog" ]
      }
    }
  }
}

filter {
  if [type] == "syslog" {
    mutate {
      remove_tag => "Ready"
    }
  }
}

#     match => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
filter {
  if "syslog" in [tags] {
    grok {
      match => { "message" => "(:\[%{POSINT:syslog_pri}\])?%{SYSLOGTIMESTAMP:syslog_timestamp} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    syslog_pri { }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM  dd HH:mm:ss" ]
      locale => "en"
    }

    if !("_grokparsefailure" in [tags]) {
      mutate {
        replace => [ "@source_host", "%{syslog_hostname}" ]
        replace => [ "@message", "%{syslog_message}" ]
      }
    }

    mutate {
      remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ]
    }
#    if "_grokparsefailure" in [tags] {
#      drop { }
#    }
  }
}

11-pfsense.conf

# from http://elijahpaul.co.uk/monitoring-pfsense-2-1-logs-using-elk-logstash-kibana-elasticsearch/

filter {
      #change to pfSense ip address
      if [host] =~ /10\.55\.0\.1/ {
          grok {
                  add_tag => [ "firewall" ]
                  match => [ "message", "<(?<evtid>.*)>(?<datetime>(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) (?:2[0123]|[01]?[0-9]):(?:[0-5][0-9]):(?:[0-5][0-9])) (?<prog>.*?): (?<msg>.*)" ]
              }
              mutate {
                  gsub => ["datetime","  "," "]
              }
              date {
                  match => [ "datetime", "MMM dd HH:mm:ss" ]
              timezone => "Europe/London"
              }
              mutate {
                  replace => [ "message", "%{msg}" ]
              }
              mutate {
                  remove_field => [ "msg", "datetime" ]
              }
          }
          if [prog] =~ /^pf$/ {
              mutate {
                  add_tag => [ "packetfilter" ]
              }
              multiline {
                  pattern => "^\s+|^\t\s+"
                  what => "previous"
              }
              mutate {
                  remove_field => [ "msg", "datetime" ]
                  remove_tag => [ "multiline" ]
              }
              grok {
                  match => [ "message", "rule (?<rule>.*)\(.*\): (?<action>pass|block) (?<direction>in|out).* on (?<iface>.*): .* proto (?<proto>TCP|UDP|IGMP|ICMP) .*\n\s*(?<src_ip>(\d+\.\d+\.\d+\.\d+))\.?(?<src_port>(\d*)) [<|>] (?<dest_ip>(\d+\.\d+\.\d+\.\d+))\.?(?<dest_port>(\d*)):" ]
              }
              if [prog] =~ /^dhcpd$/ {
              if [message] =~ /^DHCPACK|^DHCPREQUEST|^DHCPOFFER/ {
                  grok {
                      match => [ "message", "(?<action>.*) (on|for|to) (?<src_ip>[0-2]?[0-9]?[0-9]\.[0-2]?[0-9]?[0-9]\.[0-2]?[0-9]?[0-9]\.[0-2]?[0-9]?[0-9]) .*(?<mac_address>[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]).* via (?<iface>.*)" ]
                  }
              }
              if [message] =~ /^DHCPDISCOVER/ {
                  grok {
                      match => [ "message", "(?<action>.*) from (?<mac_address>[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]:[0-9a-fA-F][0-9a-fA-F]).* via (?<iface>.*)" ]
                  }
              }
              if [message] =~ /^DHCPINFORM/ {
                  grok {
                      match => [ "message", "(?<action>.*) from (?<src_ip>.*).* via (?<iface>.*)" ]
                  }
              }
          }
      }
  # from something similar to https://community.ulyaoth.net/threads/how-to-create-a-logstash-geoip-based-dashboard.29/
  if [src_ip]  {
    geoip {
      source => "src_ip"
      target => "geoip"
      add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
      add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
    }
    mutate {
      convert => [ "[geoip][coordinates]", "float" ]
    }
  }
}    

12-apache.conf

filter {
  if [type] == "apache" {
    grok {
      # See the following URL for a complete list of named patterns
      # logstash/grok ships with by default:
      # https://github.com/logstash/logstash/tree/master/patterns
      #
      # The grok filter will use the below pattern and on successful match use
      # any captured values as new fields in the event.
      match => { "message" => "%{COMBINEDAPACHELOG}" }
    }

    date {
      # Try to pull the timestamp from the 'timestamp' field (parsed above with
      # grok). The apache time format looks like: "18/Aug/2011:05:44:34 -0700"
      match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
    }

    # from something similar to https://community.ulyaoth.net/threads/how-to-create-a-logstash-geoip-based-dashboard.29/
    if [clientip]  {
      geoip {
        source => "clientip"
        target => "geoip"
        add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
        add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
      }
      mutate {
        convert => [ "[geoip][coordinates]", "float" ]
      }
    }

  }
}

13-nginx.conf

filter {
  if [log_type] == "access" {
      grok {
          patterns_dir => "/usr/local/etc/logstash/conf.d/patterns"
          pattern => "%{NGINXACCESS}"
      }
      date {
          locale => "en"
          match => [ "time_local", "dd/MMM/YYYY:HH:mm:ss Z" ]
      }
  }
}

20-tapes.conf

filter {
  if [type] == "tapes" {
    mutate {
       add_tag => ["tapes"]
    }
  }
}

filter {
  if "tapes" in [tags] {
    csv {
      # the next line contains a TAB in the "quotes"
      separator => "	"
      columns => ["date", "volume", "errors"]
   }

#    grok {
#      match => [ "message", "(?<timestamp>.*)\t(?<volume>.*)\t(?<errors>.*)" ]
#    }
  }
}

99-outputs.conf

output {
  elasticsearch { host  => "localhost" } 
  stdout        { codec => rubydebug }

  file {
    path => "/var/log/logstash/logstash.stdout"
  }
}

This is logstash 1.5.1 on FreeBSD 10.1-RELEASE-p15

Hmm, ok so two things - I think the file output not being "rotation" aware is somewhat expected behavior, though I'm open to improvements.

Two, I think /var/log/logstash.log not being rotated is a bug. Are you using logstash from ports? I can try to reproduce.

Yes, I am using logstash from ports, but I build my own via poudriere. Let me check for any options I use.

No local options. Just the default for building logstash.

OK awesome, I'll check it out.

I filed elastic/logstash#3647 to track this and linked to this gist.