Full logstash config with grok patterns. Grok pattern for Nginx, Tomcat, Spring

Readme.md

Below you can see the full work example for logstash.conf file

There are grok patterns for:
Nginx:
* access.log
* error.log
Tomcat:
* site.log
* localhost.log
* site_access_log.txt
* internal_access_log.txt
Spring:
* spring.log (for Spring's standard config)


If something wouldnt work for your case, you could check in 'debug' mode for logstash, how to run it you could find here: https://stackoverflow.com/a/67371990/9209083

custom_patterns

#/etc/logstash/patterns/custom_patterns
MYDATA (.*(\r|\n|\\r|\\n))*
TOMCAT_LOGLEVEL ([A-a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?|CONFIG|FINE|FINER|FINEST)

logstash.conf

input {
  beats {
    port => 5044
    ssl => false
  }
}
filter {

############################# NGINX ##############################
  if [event][module] == "nginx" {

########## access.log ##########
    if [fileset][name] == "access" {
      grok {
        match => { "message" => ["%{IPORHOST:ip} - %{DATA:user_name} \[%{HTTPDATE:time}\] \"%{WORD:http_method} %{DATA:url} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:body_sent_bytes} \"%{DATA:referrer}\" \"%{DATA:agent}\""] }
        remove_field => "message"
      }
      date {
        match => ["time", "dd/MMM/YYYY:HH:mm:ss Z"]
        target => "@timestamp"
        remove_field => "time"
      }
      useragent {
        source => "agent"
        target => "user_agent"
        remove_field => "agent"
      }
      geoip {
        source => "ip"
        target => "geoip"
      }
    }

########## error.log ##########
    else if [fileset][name] == "error" {
      grok {
        match => { "message" => ["%{DATA:time} \[%{DATA:log_level}\] %{NUMBER:pid}#%{NUMBER:tid}: (\*%{NUMBER:connection_id} )?%{GREEDYDATA:messageTmp}"] }
        remove_field => "message"
      }
      date {
        match => ["time", "YYYY/MM/dd HH:mm:ss"]
        target => "@timestamp"
        remove_field => "time"
      }

      mutate {
        rename => {"messageTmp" => "message"}
      }
    }

    grok {
      remove_field => "[event]"
    }

    mutate {
      add_field => {"serviceName" => "nginx"}
    }
  }
############################## TOMCAT ##############################
  else if [log][file][path] =~ ".*\\Tomcat.*" {

########## site.log ##########
    if [log][file][path] =~ ".*\\site\..*\.log" {
      grok {
        patterns_dir => ["/etc/logstash/patterns"]
        match => {"message" => "%{DATA:time}\s+%{TOMCAT_LOGLEVEL:log_level}\s+\[%{DATA:thread}\]\s+%{DATA:class}\s+%{MYDATA:messageTmp}"}
        remove_field => "message"
      }

      date {
        match => ["time", "dd-MMM-YYYY HH:mm:ss.SSS"]
        target => "@timestamp"
        remove_field => "time"
      }

      mutate {
        rename => {"messageTmp" => "message"}
      }

    }

########## localhost.log ##########
    else if [log][file][path] =~ ".*\\localhost\..*\.log" {
      grok {
        patterns_dir => ["/etc/logstash/patterns"]
        match => {"message" => "%{DATA:time}\s+%{TOMCAT_LOGLEVEL:log_level}\s+\[%{DATA:thread}\]\s+%{DATA:class}\s+%{MYDATA:messageTmp}"}
        remove_field => "message"
      }

      date {
        match => ["time", "dd-MMM-YYYY HH:mm:ss.SSS"]
        target => "@timestamp"
        remove_field => "time"
      }

      mutate {
        rename => {"messageTmp" => "message"}
      }
    }
########## site_access_log.txt ##########
    else if [log][file][path] =~ ".*\\site_access_log\..*\.txt" {
      grok {
        match => {"message" => "%{IPV4:ip} \-\ \-\ \[%{HTTPDATE:time}\] \"%{WORD:http_method} %{DATA:url} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:body_sent_bytes}"}
        remove_field => "message"
      }

      date {
        match => ["time", "dd/MMM/YYYY:HH:mm:ss Z"]
        target => "@timestamp"
        remove_field => "time"
      }
    }
########## internal_access_log.txt ##########
    else if [log][file][path] =~ ".*\\internal_access_log\..*\.txt" {
      grok {
        match => {"message" => "%{IPV4:ip} \-\ \-\ \[%{HTTPDATE:time}\] \"%{WORD:http_method} %{DATA:url} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:body_sent_bytes}"}
        remove_field => "message"
      }

      date {
        match => ["time", "dd/MMM/YYYY:HH:mm:ss Z"]
        target => "@timestamp"
        remove_field => "time"
      }
    }

    mutate {
      add_field => {"serviceName" => "back"}
    }
  }
############################## Microservices ##############################
  else {
    grok {
      match => { "message" => "%{TIMESTAMP_ISO8601:time}\s+%{LOGLEVEL:log_level}\s+\[%{DATA:appName},%{DATA:traceId},%{DATA:spanId},%{DATA:exportable}\]\s+%{DATA:pid}\s+---\s+\[%{DATA:thread}\]\s+%{DATA:class}\s+:\s+%{GREEDYDATA:messageTmp}" }
#      remove_field => "message"
    }

    date {
      match => ["time", "YYYY-MM-dd HH:mm:ss.SSS"]
      target => "@timestamp"
      remove_field => "time"
    }

    mutate {
      add_field => {"serviceName" => "back"}
      rename => {"messageTmp" => "message"}
    }
  }
############################## Global settings ##############################
  if [@metadata][ip_address] == "192.168.0.1" {
    mutate {
      add_field => {"serverName" => "back_dev"}
    }
  }
  else if [@metadata][ip_address] == '192.168.0.2' {
    mutate {
      add_field => {"serverName" => "back_prod"}
    }
  }
  else if [@metadata][ip_address] == '192.168.0.3' {
    mutate {
      add_field => {"serverName" => "front_dev"}
    }
  }
  else if [@metadata][ip_address] == '192.168.0.4' {
    mutate {
      add_field => {"serverName" => "front_prod"}
    }
  }
  
  fingerprint {
    source => "message"
    target => "[@metadata][fingerprint]"
    method => "MURMUR3"
  }
}

output {
  elasticsearch {
    hosts => ["localhost:9200"]
    manage_template => false
    index => "%{serverName}-%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_id => "%{[@metadata][fingerprint]}"
  }

  if [log_level] in ["error", "ERROR", "fatal", "FATAL", "severe", "SEVERE"] {
    email {
      from => "[email protected]"
      to => "[email protected]"
      subject => "From logstash %{log_level} - %{serverName} -> %{serviceName} -> %{appName}"
      body => "%{message}"

      codec => "plain"
      contenttype => "text/plain; charset=UTF-8"
      address => "smtp.gmail.com"
      port => "587"
      authentication => "plain"
      username => "[email protected]"
      password => "myPass"
      use_tls => true
      debug => true
    }
  }

  stdout {
    codec => rubydebug {
      metadata => true
    }
  }
}