Fortigate 300C logstash log parsing

README.md

Fortigate 300c log parsing in Logstash

QA ca be done with:

logstash-1.4.2/bin/logstash rspec --format documentation test.rb

filter.conf

#
# Configure syslog filtering
# for the Fortigate firewall logs
#
filter {
		mutate {
			add_tag => ["fortigate"]
			add_field => [ "zabbix_host", "fw.hq.aoe.lan" ]
		}
		grok {
			match => ["message", "%{SYSLOG5424PRI:syslog_index}%{GREEDYDATA:message}"]
			overwrite => [ "message" ]
			tag_on_failure => [ "failure_grok_fortigate" ]
		}

		kv { }

		if [msg] {
			mutate {
				replace => [ "message", "%{msg}" ]
			}
		}

		mutate {
			add_field => ["logTimestamp", "%{date} %{time}"]
			add_field => ["loglevel", "%{level}"]
			replace => [ "fortigate_type", "%{type}"]
			replace => [ "fortigate_subtype", "%{subtype}"]
			remove_field => [ "msg", "type", "level", "date", "time" ]
		}
		date {
			locale => "en"
			match => ["logTimestamp", "YYYY-MM-dd HH:mm:ss"]
			remove_field => ["logTimestamp", "year", "month", "day", "time", "date"]
			add_field => ["type", "syslog"]
		}
}

test.rb

# encoding: UTF-8

###############################################
# fortigate_spec.rb - Unit tests which make sure
# we can parse Firewall logs
###############################################

require "test_utils"
require_relative "config_fetcher"
describe "Fortigate logs" do
  extend LogStash::RSpec
  conf = IO.read("./filter.conf")
  config conf

  logs = <<-EOF
<187>date=2015-04-10 time=20:26:33 devname=FG300C3913606597 devid=FG300C3913606597 logid=0101037128 type=event subtype=vpn level=error vd="root" msg="progress IPsec phase 1" action=negotiate remip=1270.0.2 locip=1270.0.1 remport=1659 locport=4500 outintf="port10" cookies="38c1bf7739f47688/0000000000000000" user="N/A" group="N/A" xauthuser="N/A" xauthgroup="N/A" vpntunnel="N/A" status=failure init=remote mode=main dir=inbound stage=1 role=responder result=ERROR
<187>date=2015-04-10 time=20:26:33 devname=FG300C3913606597 devid=FG300C3913606597 logid=0101037124 type=event subtype=vpn level=error vd="root" msg="IPsec phase 1 error" action=negotiate remip=1270.0.2 locip=1270.0.1 remport=1659 locport=4500 outintf="port10" cookies="38c1bf7739f47688/0000000000000000" user="N/A" group="N/A" xauthuser="N/A" xauthgroup="N/A" vpntunnel="N/A" status=negotiate_error error_reason="peer SA proposal not match local policy" peer_notif="NOT-APPLICABLE"
<185>date=2015-04-10 time=15:44:45 devname=FG300C3913606597 devid=FG300C3913606597 logid=0101039426 type=event subtype=vpn level=alert vd="root" action="ssl-login-fail" tunneltype="ssl-web" tunnel_id=0 remote_ip=1270.0.1 tunnel_ip=(null) user="jens.weber" group="SSLVPN Tunnel Users" dst_host="N/A" reason="no_matching_policy" msg="SSL user failed to logged in"
  EOF
  samples = logs.split("\n").inject([]) { |samples, msg|
    samples.push({'message' => msg, 'host' => '10.99.0.1', 'type' => "syslog"})
    samples
  }

  sample(samples) do
    #puts "-RESULT-\n" + JSON.pretty_generate(subject) + "\n--"

    insist { subject.class } == Array
    insist { subject.size } == 3
    for i in 0..2 do
      reject { subject[i]['tags'] }.include? 'failure_grok_fortigate'
      insist { subject[i]['type'] } == 'syslog'
    end
  end
end