nginx_log-filebeat-elk.md

Topology

nginx access.log >> filebeat >> logstash >> elasticsearch

Agent

file /etc/filebeat/filebeat.yml

#=========================== Filebeat inputs =============================

filebeat.inputs:

# Each - is an input. Most options can be set at the input level, so
# you can use different inputs for various configurations.
# Below are the input specific configurations.

- type: log

  # Change to true to enable this input configuration.
  enabled: true

  # Paths that should be crawled and fetched. Glob based paths.
  paths:
    - /var/log/nginx/nganu.web.id.access.log

#----------------------------- Logstash output --------------------------------
output.logstash:
  # The Logstash hosts
  hosts: ["112.x.x.x:5454"]

ELK Server

file /etc/logstash/conf.d/11-nginx-access-log.conf

input {
    beats {
        host => "0.0.0.0"
        port => 5454
        type => "NGINX"
    }
}


filter {
 grok {
   match => [ "message" , "%{COMBINEDAPACHELOG}+%{GREEDYDATA:extra_fields}"]
   overwrite => [ "message" ]
 }
 mutate {
   convert => ["response", "integer"]
   convert => ["bytes", "integer"]
   convert => ["responsetime", "float"]
 }
 geoip {
   source => "clientip"
   target => "geoip"
   add_tag => [ "nginx-geoip" ]
 }
 date {
   match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
   remove_field => [ "timestamp" ]
 }

 useragent {
   source => "message"
    }
}



output {
if [type] == "NGINX" {
 elasticsearch {
   hosts => ["127.0.0.1:9200"]
   index => "nginx-accesslog-%{+YYYY.MM.dd}"
   document_type => "nginx_logs"
 }
}
}