Cloud.gov.au web log config for logstash

logstash.conf

input {
	tcp {
		port => 5000
	}
}

filter {
  json {
    source => "message"
  }

  grok {
    match => { "msg" => "%{HOSTNAME:[rtr][hostname]} - \[(?<rtr_time>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}T%{TIME}+%{INT})\] \"%{WORD:[rtr][verb]} %{URIPATHPARAM:[rtr][path]} %{PROG:[rtr][http_spec]}\" %{BASE10NUM:[rtr][status]:int} %{BASE10NUM:[rtr][request_bytes_received]:int} %{BASE10NUM:[rtr][body_bytes_sent]:int} \"%{GREEDYDATA:[rtr][referer]}\" \"%{GREEDYDATA:[rtr][http_user_agent]}\" \"%{IPORHOST:[rtr][src][host]}:%{POSINT:[rtr][src][port]:int}\" \"%{IPORHOST:[rtr][dst][host]}:%{POSINT:[rtr][dst][port]:int}\" x_forwarded_for:\"%{GREEDYDATA:[rtr][x_forwarded_for]}\" x_forwarded_proto:\"%{GREEDYDATA:[rtr][x_forwarded_proto]}\" vcap_request_id:\"%{NOTSPACE:[rtr][vcap_request_id]}\" response_time:%{NUMBER:[rtr][response_time_sec]:float} app_id:\"%{NOTSPACE:[rtr][app][id]}\" app_index:\"%{BASE10NUM:[rtr][app][index]:int}\" x_b3_traceid:\"%{NOTSPACE:[rtr][x_b3_traceid]}\" x_b3_spanid:\"%{NOTSPACE:[rtr][x_b3_spanid]}\" x_b3_parentspanid:\"%{NOTSPACE:[rtr][x_b3_parentspanid]}\"" }

    tag_on_failure => [ "fail/cloudfoundry/app-rtr/grok" ]
  }

  # extract tokens
  if "reset-password" in [rtr][path] and "framework" not in [rtr][path] {
    grok {
      match => { "[rtr][path]" => "/api/2/reset-password/%{NOTSPACE:[rtr][reset_token]}" }
    }
  }

  mutate {
    add_field => { "[rtr][response_time_ms]" => "%{[rtr][response_time_sec]}000" }
  }
  mutate {
    gsub => ["[rtr][response_time_ms]", "\.(\d)(\d)(\d)([\d]{0,3}).*","\1\2\3.\4"]
  }
  mutate {
    convert => { "[rtr][response_time_ms]" => "float" }
  }

  # Set [rtr][x_forwarded_for]
  mutate {
    gsub => ["[rtr][x_forwarded_for]","[\s\"]",""] # remove quotes and whitespace
    split => ["[rtr][x_forwarded_for]", ","] # format is client, proxy1, proxy2 ...
  }

  # Set [rtr][remote_addr]
  mutate {
    add_field => ["[rtr][remote_addr]", "%{[rtr][x_forwarded_for][0]}"]
  }
  if [rtr][remote_addr] =~ /([0-9]{1,3}\.){3}[0-9]{1,3}/ {
    geoip {
      source => "[rtr][remote_addr]"
    }
  }

  mutate {
    replace => {"msg" => "%{[rtr][status]} %{[rtr][verb]} %{[rtr][path]} (%{[rtr][response_time_ms]} ms)"}
  }
}

output {
	elasticsearch {
		hosts => "elasticsearch:9200"
	}
}