Fortigate FortiOS 5.2 (and 5.2.2) Logstash Grok patterns

fortigate52.pattern

FORTIDATE %{YEAR:year}\-%{MONTHNUM:month}\-%{MONTHDAY:day}
FORTIGATE_52BASE <%{NUMBER:syslog_index}>date=%{FORTIDATE:date} time=%{TIME:time} devname=%{HOST:hostname} devid=%{HOST:devid} logid=%{NUMBER:logid} type=%{WORD:type} subtype=%{WORD:subtype} eventtype=%{WORD:eventtype} level=%{WORD:level} vd=\"%{WORD:vdom}\"
FORTIGATE_52BASEV2 <%{NUMBER:syslog_index}>date=%{FORTIDATE:date} time=%{TIME:time} devname=%{HOST:hostname} devid=%{HOST:devid} logid=%{NUMBER:logid} type=%{WORD:type} subtype=%{WORD:subtype} level=%{WORD:level} vd=\"%{WORD:vdom}\"
FORTIGATE_52IPS severity=%{WORD:severity} srcip=%{IP:srcip} dstip=%{IP:dstip} sessionid=%{NUMBER:sessionid} action=%{DATA:action} proto=%{NUMBER:proto} service=%{DATA:service} attack="%{DATA:attack}" srcport=%{NUMBER:srcport} dstport=%{NUMBER:dstport} direction=%{NUMBER:direction} attackid=%{NUMBER:attackid} profile=\"%{DATA:profile}\" ref=\"%{DATA:ref}\";? incidentserialno=%{NUMBER:incidentserialno} msg=\"%{GREEDYDATA:msg}\"
FORTIGATE_52DOS severity=%{WORD:severity} srcip=%{IP:srcip} dstip=%{IP:dstip} sessionid=%{NUMBER:sessionid} action=%{DATA:action} proto=%{NUMBER:proto} service=%{DATA:service} srcintf=\"%{HOST:srcintf}\" count=%{NUMBER:count} attack=\"%{DATA:attack}\" srcport=%{NUMBER:srcport} dstport=%{NUMBER:dstport} direction=%{NUMBER:direction} attackid=%{NUMBER:attackid} profile=\"%{DATA:profile}\" ref=\"%{DATA:ref}\";? msg=\"%{GREEDYDATA:msg}\" crscore=%{NUMBER:crscore} craction=%{NUMBER:craction}

It seems the syslog format has been updated again for FortiOS 5.4. I am working on grok patterns for these, but will likely take me more time as I haven't gotten one message to parse properly in logstash (using http://grokconstructor.appspot.com/ has helped a lot, but still won't work).
I'll post results.

I've found that using Grok patterns is inefficient (assuming this is for logstash). Using the kv filter worked in a snap:

filter {
 kv {
      source => "message"
        exclude_keys => [ "type", "subtype" ] }
        geoip { source => "dst" }
        geoip { source => "dstip" }
        geoip { source => "src" }
        geoip { source => "srcip" }

        mutate {

            rename => [ "dst", "dst_ip" ]
            rename => [ "dstip", "dst_ip" ]
            rename => [ "dstport", "dst_port" ]
            rename => [ "devname", "device_id" ]
            rename => [ "status", "action" ]
            rename => [ "src", "src_ip" ]
            rename => [ "srcip", "src_ip" ]
            rename => [ "zone", "src_intf" ]
            rename => [ "srcintf", "src_intf" ]
            rename => [ "srcport", "src_port" ]
            rename => [ "rcvd", "byte_recieved" ]
            rename => [ "rcvdbyte", "bytes_recieved" ]
            rename => [ "sentbyte", "bytes_sent" ]
            rename => [ "sent", "bytes_sent" ]
            convert => ["bytes_recieved", "integer"]
            convert => ["bytes_sent", "integer"]
            remove_field => [ "msg" ]
    }
  }
}

Hi Blandman,

Thanks for your filter for fortigate 5.4 OS. But while fetching the logs to logstash in front of log as message:
<29>date=2016-09-17 time=11:48:40 how to filter the ID.

dsara35: Use the logstash-core 'SYSLOG5424PRI' pattern to strip the prefix as shown in tolleiv's gist.

Filter:

filter {
  grok {
    patterns_dir => ["/etc/logstash/patterns"]
    match => [ "message", "%{SYSLOG5424PRI}%{GREEDYDATA:raw_message}" ]
  }
  # Perform kv filtering using 'raw_message' key
}

I've found that using Grok patterns is inefficient (assuming this is for logstash). Using the kv filter worked in a snap:

filter {
 kv {
      source => "message"
        exclude_keys => [ "type", "subtype" ] }
        geoip { source => "dst" }
        geoip { source => "dstip" }
        geoip { source => "src" }
        geoip { source => "srcip" }

        mutate {

            rename => [ "dst", "dst_ip" ]
            rename => [ "dstip", "dst_ip" ]
            rename => [ "dstport", "dst_port" ]
            rename => [ "devname", "device_id" ]
            rename => [ "status", "action" ]
            rename => [ "src", "src_ip" ]
            rename => [ "srcip", "src_ip" ]
            rename => [ "zone", "src_intf" ]
            rename => [ "srcintf", "src_intf" ]
            rename => [ "srcport", "src_port" ]
            rename => [ "rcvd", "byte_recieved" ]
            rename => [ "rcvdbyte", "bytes_recieved" ]
            rename => [ "sentbyte", "bytes_sent" ]
            rename => [ "sent", "bytes_sent" ]
            convert => ["bytes_recieved", "integer"]
            convert => ["bytes_sent", "integer"]
            remove_field => [ "msg" ]
    }
  }
}

When I use your filter, my port 5044 will be down and I get :

warning: method redefined; discarding old to_int
warning: method redefined; discarding old to_f