Skip to content

Instantly share code, notes, and snippets.

@rmrf-run

rmrf-run/13-filter-bro.json

Last active December 22, 2017 06:26
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save rmrf-run/bde080f7b0d105d59d0d to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save rmrf-run/bde080f7b0d105d59d0d to your computer and use it in GitHub Desktop.
Download ZIP
Bro IDS - logstash-forwarder - Kibana Dashboard - Bro grok filter
Raw
13-filter-bro.json
filter {
if [message] =~ /^#/ {
drop { }
}
else {
if [type] == "BRO_httplog" {
grok {
match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id.orig_h>(.*?))\t(?<id.orig_p>(.*?))\t(?<id.resp_h>(.*?))\t(?<id.resp_p>(.*?))\t(?<trans_depth>(.*?))\t(?<method>(.*?))\t(?<host>(.*?))\t(?<uri>(.*?))\t(?<referrer>(.*?))\t(?<user_agent>(.*?))\t(?<request_body_len>(.*?))\t(?<response_body_len>(.*?))\t(?<status_code>(.*?))\t(?<status_msg>(.*?))\t(?<info_code>(.*?))\t(?<info_msg>(.*?))\t(?<filename>(.*?))\t(?<tags>(.*?))\t(?<username>(.*?))\t(?<password>(.*?))\t(?<proxied>(.*?))\t(?<orig_fuids>(.*?))\t(?<orig_mime_types>(.*?))\t(?<resp_fuids>(.*?))\t(?<resp_mime_types>(.*))" ]
}
}
if [type] == "BRO_ssh" {
grok {
match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id.orig_h>(.*?))\t(?<id.orig_p>(.*?))\t(?<id.resp_h>(.*?))\t(?<id.resp_p>(.*?))\t(?<status>(.*?))\t(?<direction>(.*?))\t(?<client>(.*?))\t(?<server>(.*?))\t(?<remote_location.country_code>(.*?))\t(?<remote_location.region>(.*?))\t(?<remote_location.city>(.*?))\t(?<remote_location.latitude>(.*?))\t(?<remote_location.longitude>(.*))" ]
}
}
if [type] == "BRO_connlog" {
grok {
match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id.orig_h>(.*?))\t(?<id.orig_p>(.*?))\t(?<id.resp_h>(.*?))\t(?<id.resp_p>(.*?))\t(?<proto>(.*?))\t(?<service>(.*?))\t(?<duration>(.*?))\t(?<orig_bytes>(.*?))\t(?<resp_bytes>(.*?))\t(?<conn_state>(.*?))\t(?<local_orig>(.*?))\t(?<missed_bytes>(.*?))\t(?<history>(.*?))\t(?<orig_pkts>(.*?))\t(?<orig_ip_bytes>(.*?))\t(?<resp_pkts>(.*?))\t(?<resp_ip_bytes>(.*?))\t(?<tunnel_parents>(.*))" ]
}
}
if [type] == "BRO_weirdlog" {
grok {
match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id.orig_h>(.*?))\t(?<id.orig_p>(.*?))\t(?<id.resp_h>(.*?))\t(?<id.resp_p>(.*?))\t(?<name>(.*?))\t(?<addl>(.*?))\t(?<notice>(.*?))\t(?<peer>(.*))" ]
}
}
if [type] == "BRO_dhcp" {
grok {
match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id.orig_h>(.*?))\t(?<id.orig_p>(.*?))\t(?<id.resp_h>(.*?))\t(?<id.resp_p>(.*?))\t(?<mac>(.*?))\t(?<assigned_ip>(.*?))\t(?<lease_time>(.*?))\t(?<trans_id>(.*))" ]
}
}
if [type] == "BRO_files" {
grok {
match => [ "message", "(?<ts>(.*?))\t(?<fuid>(.*?))\t(?<tx_hosts>(.*?))\t(?<rx_hosts>(.*?))\t(?<conn_uids>(.*?))\t(?<source>(.*?))\t(?<depth>(.*?))\t(?<analyzers>(.*?))\t(?<mime_type>(.*?))\t(?<filename>(.*?))\t(?<duration>(.*?))\t(?<local_orig>(.*?))\t(?<is_orig>(.*?))\t(?<seen_bytes>(.*?))\t(?<total_bytes>(.*?))\t(?<missing_bytes>(.*?))\t(?<overflow_bytes>(.*?))\t(?<timedout>(.*?))\t(?<parent_fuid>(.*?))\t(?<md5>(.*?))\t(?<sha1>(.*?))\t(?<sha256>(.*?))\t(?<extracted>(.*))" ]
}
}
if [type] == "BRO_SSL" {
grok {
match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id.orig_h>(.*?))\t(?<id.orig_p>(.*?))\t(?<id.resp_h>(.*?))\t(?<id.resp_p>(.*?))\t(?<version>(.*?))\t(?<cipher>(.*?))\t(?<server_name>(.*?))\t(?<session_id>(.*?))\t(?<subject>(.*?))\t(?<issuer_subject>(.*?))\t(?<not_valid_before>(.*?))\t(?<not_valid_after>(.*?))\t(?<last_alert>(.*?))\t(?<client_subject>(.*?))\t(?<client_issuer_subject>(.*?))\t(?<cert_hash>(.*?))\t(?<validation_status>(.*))" ]
}
}
if [type] == "BRO_notice" {
grok {
match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id.orig_h>(.*?))\t(?<id.orig_p>(.*?))\t(?<id.resp_h>(.*?))\t(?<id.resp_p>(.*?))\t(?<fuid>(.*?))\t(?<file_mime_type>(.*?))\t(?<file_desc>(.*?))\t(?<proto>(.*?))\t(?<note>(.*?))\t(?<msg>(.*?))\t(?<sub>(.*?))\t(?<src>(.*?))\t(?<dst>(.*?))\t(?<p>(.*?))\t(?<n>(.*?))\t(?<peer_descr>(.*?))\t(?<actions>(.*?))\t(?<suppress_for>(.*?))\t(?<dropped>(.*?))\t(?<remote_location.country_code>(.*?))\t(?<remote_location.region>(.*?))\t(?<remote_location.city>(.*?))\t(?<remote_location.latitude>(.*?))\t(?<remote_location.longitude>(.*))" ]
}
}
if [type] == "BRO_dns" {
grok {
match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id.orig_h>(.*?))\t(?<id.orig_p>(.*?))\t(?<id.resp_h>(.*?))\t(?<id.resp_p>(.*?))\t(?<proto>(.*?))\t(?<trans_id>(.*?))\t(?<query>(.*?))\t(?<qclass>(.*?))\t(?<qclass_name>(.*?))\t(?<qtype>(.*?))\t(?<qtype_name>(.*?))\t(?<rcode>(.*?))\t(?<rcode_name>(.*?))\t(?<AA>(.*?))\t(?<TC>(.*?))\t(?<RD>(.*?))\t(?<RA>(.*?))\t(?<Z>(.*?))\t(?<answers>(.*?))\t(?<TTLs>(.*?))\t(?<rejected>(.*))" ]
}
}
if [id.orig_h] {
geoip {
source => "id.orig_h"
target => "geoip"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
}
mutate {
uppercase => [ "geoip.country_code2" ]
}
}
}
}
Raw
bro-kibana-dashboard.json
{
"title": "Traffic Across the IDS",
"services": {
"query": {
"idQueue": [],
"list": {
"0": {
"query": "conn.log",
"alias": "",
"color": "#7EB26D",
"id": 0,
"pin": false,
"type": "lucene",
"enable": true
},
"1": {
"query": "weird.log",
"alias": "",
"color": "#EAB839",
"pin": false,
"id": 1,
"type": "lucene",
"enable": true
},
"2": {
"query": "ssl.log",
"alias": "",
"color": "#6ED0E0",
"pin": false,
"id": 2,
"type": "lucene",
"enable": true
},
"3": {
"query": "dns.log",
"alias": "",
"color": "#EF843C",
"pin": false,
"id": 3,
"type": "lucene",
"enable": true
},
"5": {
"id": 5,
"color": "#1F78C1",
"alias": "",
"pin": false,
"type": "lucene",
"enable": true,
"query": "ssh.log"
}
},
"ids": [
0,
1,
2,
3,
5
]
},
"filter": {
"idQueue": [
1,
2
],
"list": {
"0": {
"type": "time",
"field": "@timestamp",
"from": "now-2d",
"to": "now",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
}
},
"ids": [
0
]
}
},
"rows": [
{
"title": "Query",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 10,
"editable": true,
"type": "bettermap",
"loadingEditor": false,
"field": "geoip.coordinates",
"size": 1000,
"spyable": true,
"tooltip": "_id",
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
5
]
},
"title": "Map of stuff"
},
{
"span": 2,
"editable": true,
"type": "trends",
"loadingEditor": false,
"ago": "30m",
"arrangement": "vertical",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
5
]
},
"style": {
"font-size": "14pt"
},
"title": "Trends over past 30m"
}
],
"notice": false
},
{
"title": "Graph",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 4,
"editable": true,
"group": [
"default"
],
"type": "histogram",
"mode": "count",
"time_field": "@timestamp",
"value_field": null,
"auto_int": true,
"resolution": 100,
"interval": "30m",
"fill": 3,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": true,
"stack": true,
"points": false,
"lines": false,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
5
]
},
"title": "Events over time",
"scale": 1,
"y_format": "none",
"grid": {
"max": null,
"min": 0
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1y"
],
"pointradius": 5,
"options": true,
"show_query": true,
"legend_counts": true,
"zerofill": true,
"derivative": false,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": true
}
},
{
"span": 4,
"editable": true,
"type": "hits",
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
5
]
},
"style": {
"font-size": "10pt"
},
"arrangement": "horizontal",
"chart": "bar",
"counter_pos": "above",
"donut": false,
"tilt": false,
"labels": true,
"spyable": true,
"title": "Query hits"
},
{
"error": false,
"span": 2,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "geoip.ip",
"exclude": [],
"missing": false,
"other": false,
"size": 10,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "table",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
5
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "Top IPs"
},
{
"error": false,
"span": 2,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "geoip.country_name",
"exclude": [],
"missing": false,
"other": false,
"size": 10,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "table",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
5
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "Top Countries"
}
],
"notice": false
},
{
"title": "Events",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "table",
"size": 100,
"pages": 5,
"offset": 0,
"sort": [
"@timestamp",
"desc"
],
"style": {
"font-size": "9pt"
},
"overflow": "min-height",
"fields": [
"@timestamp",
"type",
"geoip.ip",
"message"
],
"highlight": [
"geoip.ip"
],
"sortable": true,
"header": true,
"paging": true,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
5
]
},
"field_list": true,
"status": "Stable",
"trimFactor": 300,
"normTimes": true,
"all_fields": false,
"localTime": false,
"timeField": "@timestamp"
}
],
"notice": false
}
],
"editable": true,
"index": {
"interval": "day",
"pattern": "[logstash-]YYYY.MM.DD",
"default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED",
"warm_fields": true
},
"style": "light",
"failover": false,
"panel_hints": true,
"pulldowns": [
{
"type": "query",
"collapse": false,
"notice": false,
"enable": true,
"query": "*",
"pinned": true,
"history": [
"ssh.log",
"dns.log",
"ssl.log",
"weird.log",
"conn.log",
"notice.log"
],
"remember": 10
},
{
"type": "filtering",
"collapse": false,
"notice": true,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"enable": true,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"now": true,
"filter_id": 0
}
],
"loader": {
"save_gist": false,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": false,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": false,
"hide": false
},
"refresh": false
}
Raw
logstash-forwarder.json
{
"network": {
"servers": [ "*.*.*.*:5000" ],
"timeout": 15,
"ssl ca": "/etc/pki/tls/certs/logstash-forwarder.crt"
},
"files": [
{
"paths": [
"/var/log/secure"
],
"fields": { "type": "syslog" }
},
{
"paths":[
"/var/opt/bro/spool/bro/http.log"
],
"fields": {"type": "BRO_httplog"}
},
{
"paths":[
"/var/opt/bro/spool/bro/dhcp.log"
],
"fields": {"type": "BRO_dhcp"}
},
{
"paths":[
"/var/opt/bro/spool/bro/conn.log"
],
"fields": {"type": "BRO_connlog"}
},
{
"paths":[
"/var/opt/bro/spool/bro/weird.log"
],
"fields": {"type": "BRO_weirdlog"}
},
{
"paths":[
"/var/opt/bro/spool/bro/syslog.log"
],
"fields": {"type": "BRO_syslog"}
},
{
"paths":[
"/var/opt/bro/spool/bro/ssl.log"
],
"fields": {"type": "BRO_ssl"}
},
{
"paths":[
"/var/opt/bro/spool/bro/ssh.log"
],
"fields": {"type": "BRO_ssh"}
},
{
"paths":[
"/var/opt/bro/spool/bro/snmp.log"
],
"fields": {"type": "BRO_snmp"}
},
{
"paths":[
"/var/opt/bro/spool/bro/notice.log"
],
"fields": {"type": "BRO_notice"}
},
{
"paths":[
"/var/opt/bro/spool/bro/files.log"
],
"fields": {"type": "BRO_files"}
},
{
"paths":[
"/var/opt/bro/spool/bro/dns.log"
],
"fields": {"type": "BRO_dns"}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment