Skip to content

Instantly share code, notes, and snippets.

@clay584
Created November 20, 2014 19:38
Show Gist options
  • Select an option

  • Save clay584/cb2df904ee1daa7f8ae7 to your computer and use it in GitHub Desktop.

Select an option

Save clay584/cb2df904ee1daa7f8ae7 to your computer and use it in GitHub Desktop.
logging into the future - logstash config that generated the future log
input {
file {
path => ["/var/log/network.log"]
sincedb_path => "/var/log/logstash"
start_position => "beginning"
type => "syslog"
tags => [ "netsyslog" ]
}
zeromq {
topology => "pubsub"
mode => "client"
address => "tcp://10.4.4.15:5000"
type => "nProbe"
codec => json { }
}
} #end input block
filter {
if [type] == "syslog" {
grok {
match => [ "message", "%{TIMESTAMP_ISO8601:syslog_datetime} %{IPORHOST:syslog_host} %{POSINT}?: %{WORD} %{POSINT} %{TIME}: %%{WORD:facility}-%{POSINT:severity}-%{NOTSPACE:mneumonic}: %{GREEDYDATA:raw_message}",
"message", "%{TIMESTAMP_ISO8601:syslog_datetime} %{IPORHOST:syslog_host} : %{YEAR} %{SYSLOGTIMESTAMP} %{TZ}: %%{WORD:facility}-%{POSINT:severity}-%{NOTSPACE:mneumonic}: %{GREEDYDATA:raw_message}",
"message", "%{TIMESTAMP_ISO8601:syslog_datetime} %{IPORHOST:syslog_host} : %%{WORD:facility}-%{POSINT:severity}-%{NOTSPACE:mneumonic}: %{GREEDYDATA:raw_message}",
"message", "%{TIMESTAMP_ISO8601:syslog_datetime} %{IPORHOST:syslog_host} %{IPORHOST:hostname}: \*%{WORD:category}: %{SYSLOGTIMESTAMP}: #%{WORD:facility}-%{POSINT:severity}-%{NOTSPACE:mneumonic}: %{GREEDYDATA:raw_message}",
"message", "%{TIMESTAMP_ISO8601:syslog_datetime} %{IPORHOST:syslog_host} %{IPORHOST:device_host} %{DATA:device_module}: \[Primary Detection Engine \(%{DATA:pri_detection_engine}\)\]\[%{DATA:sf_policy}\] Connection Type: %{DATA:connection_type}, User: %{DATA:user}, Client: %{DATA:client}, Application Protocol: %{DATA:app_protocol}, Web App: %{DATA:webapp}, Access Control Rule Name: %{DATA:acl_rule_name}, Access Control Rule Action: %{DATA:acl_rule_action}, Access Control Rule Reasons: %{DATA:acl_rule_reason}, URL Category: %{DATA:url_category}, URL Reputation: %{DATA:url_reputation}, URL: %{DATA:url}, Interface Ingress: %{DATA:ingress_interface}, Interface Egress: %{DATA:egress_interface}, Security Zone Ingress: %{DATA:ingress_sec_zone}, Security Zone Egress: %{DATA:egress_sec_zone}, Security Intelligence Matching IP: %{DATA:sec_intel_match_ip}, Security Intelligence Category: %{DATA:sec_intel_category}, Client Version: %{DATA:client_version}, Number of File Events: %{DATA:num_file_events:int}, Number of IPS Events: %{DATA:num_ips_events:int}, TCP Flags: %{DATA:tcp_flags}, NetBIOS Domain: %{DATA:netbios_domain}, Initiator Packets: %{DATA:initiator_packets:int}, Responder Packets: %{DATA:responder_packets:int}, Initiator Bytes: %{DATA:initiator_bytes:int}, Responder Bytes: %{DATA:responder_bytes:int}, Context: %{NOTSPACE:context} \{%{DATA:protocol}\} %{IPORHOST:source_ip}:%{POSINT:source_port} -> %{IPORHOST:dest_ip}:%{POSINT:dest_port}",
"message", "%{TIMESTAMP_ISO8601:syslog_datetime} %{IPORHOST:syslog_host} %{IPORHOST:device_host} %{DATA:device_module}: \[Primary Detection Engine \(%{DATA:pri_detection_engine}\)\]\[%{DATA:sf_policy}\] Connection Type: %{DATA:connection_type}, User: %{DATA:user}, Client: %{DATA:client}, Application Protocol: %{DATA:app_protocol}, Web App: %{DATA:webapp}, Access Control Rule Name: %{DATA:acl_rule_name}, Access Control Rule Action: %{DATA:acl_rule_action}, Access Control Rule Reasons: %{DATA:acl_rule_reason}, URL Category: %{DATA:url_category}, URL Reputation: %{DATA:url_reputation}, URL: %{DATA:url}, Interface Ingress: %{DATA:ingress_interface}, Interface Egress: %{DATA:egress_interface}, Security Zone Ingress: %{DATA:ingress_sec_zone}, Security Zone Egress: %{DATA:egress_sec_zone}, Security Intelligence Matching IP: %{DATA:sec_intel_match_ip}, Security Intelligence Category: %{DATA:sec_intel_category}, Client Version: %{DATA:client_version}, Number of File Events: %{DATA:num_file_events:int}, Number of IPS Events: %{DATA:num_ips_events:int}, TCP Flags: %{DATA:tcp_flags}, NetBIOS Domain: %{DATA:netbios_domain}, Initiator Packets: %{DATA:initiator_packets:int}, Responder Packets: %{DATA:responder_packets:int}, Initiator Bytes: %{DATA:initiator_bytes:int}, Responder Bytes: %{DATA:responder_bytes:int}, Context: %{NOTSPACE:context} \{%{DATA:protocol}\} %{IPORHOST:source_ip} -> %{IPORHOST:dest_ip}" ]
add_tag => [ "grok_ran" ]
}
date {
match => [ "syslog_datetime", "ISO8601" ]
}
}
if "grok_ran" in [tags] and "_grokparsefailure" not in [tags] {
mutate {
remove_field => [ "message", "syslog_datetime" ]
}
}
if [type] == "nProbe" {
date {
match => [ "FLOW_START_MILLISECONDS", "UNIX_MS" ]
}
}
} #end filter block
output {
elasticsearch {
protocol => "node"
host => "127.0.0.1"
cluster => "presidiolab-es"
}
} #end output block
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment