ELK configs, grok regexs and playbook

esnog19-elk-workshop

root@esnog19:/etc/logstash# cat conf.d/inputs-intermediate.conf
input {
    file {
        path => "/home/malibu/data/nokia.log"
        start_position => "beginning"
        type => "nokia"
        codec => multiline {
            pattern => "^\d+\s+\d{4}"
            what => "previous"
            negate => true
        }
    }
    file {
        path => "/home/malibu/data/junos.log"
        start_position => "beginning"
        type => "junos"
    }
}
root@esnog19:/etc/logstash# cat conf.d/filter-intermediate.conf
filter {
    if [type] == "junos" {
        grok {
            patterns_dir => ["/etc/logstash/patterns"]
            match => { "message" => "%{JUNOS_BASE}" }
            overwrite => ["message"]
        }
        date {
            # Mar 26 23:43:19
            match => ["ts", "MMM dd HH:mm:ss"]
            timezone => "UTC"
        }
        grok {
            patterns_dir => ["/etc/logstash/patterns"]
            match => { "message" => ["%{JUNOS_TCP}", "%{JUNOS_NOTIFICATION}", "%{JUNOS_AUTH_ERROR}", "%{JUNOS_SSH_LOGIN_FAILED}", "%{JUNOS_CONN}", "%{JUNOS_LOGIN}", "%{JUNOS_CONN_ATTEMPT}"] }
        }
    } else if [type] == "nokia" {
        grok {
            patterns_dir => ["/etc/logstash/patterns"]
            match => { "message" => "%{NOKIA_BASE}" }
            overwrite => ["message"]
        }
        if [logger] == "BGP #2012" {
	    grok {
                match => { "message" => "%{NOKIA_2012}" }
                patterns_dir => ["/etc/logstash/patterns"]
                add_field => {
                    "conn_state" => "closing"
                    "reason" => "not enabled in config"
                }
            }
        } else if [logger] == "BGP #2002" or [logger] == "BGP #2020" {
	    grok {
                match => { "message" => "%{NOKIA_2002_2020}" }
                patterns_dir => ["/etc/logstash/patterns"]
            }
	} else if [logger] == "BGP #2011" {
	    grok {
                match => { "message" => "%{NOKIA_2011}" }
                patterns_dir => ["/etc/logstash/patterns"]
            }
	} else if [logger] == "BGP #2006" {
	    grok {
                match => { "message" => "%{NOKIA_2006}" }
                patterns_dir => ["/etc/logstash/patterns"]
            }
	} else if [logger] == "BGP #2029" {
            grok {
                match => { "message" => "%{NOKIA_2029}" }
                patterns_dir => ["/etc/logstash/patterns"]
            }
	} else if [logger] == "BGP #2028" {
            grok {
                match => { "message" => "%{NOKIA_2028}" }
                patterns_dir => ["/etc/logstash/patterns"]
            }
	} else if [logger] == "BGP #2001" or [logger] == "BGP #2019" {
            grok {
                match => { "message" => "%{NOKIA_2001_2019}" }
                patterns_dir => ["/etc/logstash/patterns"]
            }
	} else if [logger] == "BGP #2005" {
            grok {
                match => { "message" => "%{NOKIA_2005}" }
                patterns_dir => ["/etc/logstash/patterns"]
            }
	} else if [logger] == "BGP #2009" {
            grok {
                match => { "message" => "%{NOKIA_2009}" }
                patterns_dir => ["/etc/logstash/patterns"]
            }
        }
    }
}
root@esnog19:/etc/logstash# cat conf.d/outputs-intermediate.conf
output {
    elasticsearch {
        codec => "json"
        document_type => "%{type}"
        hosts => ["esnog19.test:9200"]
        index => "esnog19-intermediate"
        #index => "esnog19-intermediate-%{type}"
        manage_template => false
    }
}
root@esnog19:/etc/logstash# cat /etc/elasticsearch/
elasticsearch.yml  jvm.options        log4j2.properties  scripts/
root@esnog19:/etc/logstash# cat /etc/elasticsearch/elasticsearch.yml
cluster.name: esnog19
node.name: esnog19.test

network.bind_host: ["10.0.2.15", "127.0.0.1"]
network.publish_host: "10.0.2.15"
root@esnog19:/etc/logstash# cat /etc/kibana/kibana.yml
server.host: "10.0.2.15"
server.name: "esnog19.test"
elasticsearch.url: "http://esnog19.test:9200"
malibu@esnog19:~$ ls -R ansible/
ansible/:
files  setup_demo.retry  setup_demo.yml  templates

ansible/files:
logstash

ansible/files/logstash:
filter-basic.conf  filter-intermediate.conf  inputs-basic.conf  inputs-intermediate.conf  outputs-basic.conf  outputs-intermediate.conf  patterns

ansible/files/logstash/patterns:
junos  nokia

ansible/templates:
elasticsearch  kibana  logstash

ansible/templates/elasticsearch:
elasticsearch.yml

ansible/templates/kibana:
kibana.yml
malibu@esnog19:~$ cat ansible/files/logstash/patterns/*
JUNOS_TS %{MONTH} %{MONTHDAY} %{TIME}

#JUNOS_BASE %{JUNOS_TS:ts}%{SPACE}+%{DATA:hostname} (?:/)?%{DATA:process}(?:\[%{INT:process_id}\])?: (%{LOGLEVEL:level}: )?%{DATA:function}:(?:%{INT:function_line}:)? %{GREEDYDATA:payload}
#JUNOS_BASE %{JUNOS_TS:ts}%{SPACE}+%{DATA:hostname} (?:/)?%{DATA:process}(?:\[%{INT:process_id}\])?: (?:%{LOGLEVEL:level}: )?(?:%{DATA:function}:(?:%{INT:function_line}:)?)? %{GREEDYDATA:payload}
JUNOS_BASE %{JUNOS_TS:ts}%{SPACE}+%{DATA:hostname} (?:/)?%{DATA:process}(?:\[%{INT:process_id}\])?: (?:%{LOGLEVEL:level}: )?(?:%{DATA:function}(?::)?(?:%{INT:function_line}:)?)? %{GREEDYDATA:message}
JUNOS_TCP Packet from %{IP:remote_addr}:%{INT:remote_port} %{GREEDYDATA:payload}
JUNOS_NOTIFICATION NOTIFICATION sent to %{IP:remote_addr}\+%{INT:remote_port} \(proto\): code %{INT:notification_code_num} \(%{DATA:notification_code}\) subcode %{INT:notification_subcode_num} \(%{DATA:notification_subcode}\)(?:,)? %{GREEDYDATA:notification_payload}
JUNOS_AUTH_ERROR PAM:(?:%{INT:function_line}:)? authentication error for %{DATA:user} from %{IP:remote_addr}
JUNOS_SSH_LOGIN_FAILED SSHD_LOGIN_FAILED: Login %{WORD:login_state} for user '%{DATA:user}' from host '%{IP:remote_addr}'
JUNOS_CONN Connection %{WORD:connection_state} by %{IP:remote_addr} \[%{WORD:connection_stage}\]
JUNOS_LOGIN %{DATA:connection_state} for %{DATA:user} from %{IP:remote_addr} port %{INT:remote_port} %{GREEDYDATA:ssh_version}
JUNOS_CONN_ATTEMPT Connection attempt from unconfigured neighbor: %{IP:remote_addr}\+%{INT:remote_port}
NOKIA_TS %{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME} %{TZ}

#NOKIA_BASE (?m)%{INT:seq}%{SPACE}+%{NOKIA_TS:ts} %{WORD:level}: (?<logger>%{WORD} #%{INT}) Base %{GREEDYDATA:payload}
NOKIA_BASE (?m)%{INT:seq}%{SPACE}+%{NOKIA_TS:ts} %{WORD:level}: (?<logger>%{WORD} #%{INT}) Base %{GREEDYDATA:message}

NOKIA_2002_2020 Peer 1: %{IP:remote_addr}\r\n\"VR %{INT:vr_id}: Group %{DATA:peergroup}: Peer %{IP}: moved from higher state %{WORD:state_from} to lower state %{WORD:state_to} due to event %{GREEDYDATA:event}\"
NOKIA_2001_2019 Peer 1: %{IP:remote_addr}\r\n\"VR %{INT:vr_id}: Group %{DATA:peergroup}: Peer %{IP}: moved into %{WORD:state_to} state\"
NOKIA_2006 Peer 1: %{IP:remote_addr}\r\n\"VR %{INT:vr_id}: Group %{DATA:peergroup}: Peer %{IP}: received notification: code %{DATA:notification_code} subcode %{DATA:notification_subcode}\"
NOKIA_2011 Peer 1: %{IP:remote_addr}\r\n\"VR %{INT:vr_id}: Group %{DATA:peergroup}: Peer %{IP}: remote end %{DATA:connection_state}\"
NOKIA_2012 PEER 1: %{IP:remote_addr}\r\n\"PEER %{INT:peer_id}: %{IP}: Closing connection: PEER 1: %{IP} not enabled or not in configuration\"
NOKIA_2029 Peer 1: %{IP:remote_addr}\r\n\"%{INT:peer_id}: BGP Peer: %{IP}, Route: (?<prefix>%{IP}/%{INT}) %{WORD:prefix_state} because of %{DATA:reason}.\"
NOKIA_2028 Peer 1: %{IP:remote_addr}\r\n\"%{INT:peer_id}: BGP Peer %{IP}: Invalid path attribute received with attribute type \[%{INT:attribute_type_num}\] and length \[%{INT:attribute_length}\]\.
NOKIA_2005 %{DATA}\n\"Interface %{DATA:iface} is %{DATA:iface_state}\"
NOKIA_2009 Peer 1: %{IP:remote_addr}\r\n\"VR %{INT:vr_id}: Group %{DATA:peergroup}: Peer %{IP}: closing inbound connection because the %{DATA:reason}\"
malibu@esnog19:~$ cat ansible/setup_demo.yml
- hosts: localhost
  tasks:
    - name: add elastic repo key
      apt_key:
        url: https://artifacts.elastic.co/GPG-KEY-elasticsearch
    - name: setup APT repo
      apt_repository:
        repo: deb https://artifacts.elastic.co/packages/5.x/apt stable main
        filename: elastic
    - name: install packages
      apt:
        update_cache: yes
        name: "{{ item }}"
      with_items:
        - openjdk-8-jdk-headless
        - logstash
        - elasticsearch
        - kibana
    - name: create logstash pattern dir
      file:
        path: /etc/logstash/patterns/
        state: directory
    - name: deploy logstash regular expressions
      copy:
        src: "files/logstash/{{ item }}"
        dest: "/etc/logstash/{{ item }}"
      with_items:
        - patterns/junos
        - patterns/nokia
    - name: deploy ELK configs
      template:
        src: "templates/{{ item }}"
        dest: "/etc/{{ item }}"
      with_items:
        - elasticsearch/elasticsearch.yml
        - kibana/kibana.yml
    - name: start services
      service:
        name: "{{ item }}"
        state: started
      with_items:
        - elasticsearch
        - kibana