Last active
April 12, 2017 11:40
-
-
Save altherlex/42539232506144d6738e to your computer and use it in GitHub Desktop.
By mining your nginx log with LOGSTASH (you'll need ELASTICSEARCH for database and KIBANA for see information)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# into Nginx server | |
# how install: https://github.com/elastic/logstash-forwarder#building-it | |
# git clone git://github.com/elasticsearch/logstash-forwarder.git | |
# cd logstash-forwarder | |
# go build // it's necessary to install go interpreter | |
# vi forwarding.conf | |
# cmd to start service> ./logstash-forwarder -config forwarding.conf | |
#### ATTENTION when you generate the certificate | |
# the name on CN flag into comand openssl will be the same of servers configuration into file | |
# exemple: | |
# cmd: openssl req -x509 -subj '/CN=logstashserver.mycompany.com:5000/' -nodes -newkey rsa:2048 -keyout logstash-forwarder.key -out logstash-forwarder.crt | |
# see more: https://github.com/elastic/logstash-forwarder#important-tlsssl-certificate-notes | |
{ | |
"network": { | |
"servers": [ "logstashserver.mycompany.com:5000" ], | |
"ssl certificate": "/usr/local/etc/logstash-forwarder/logstash-forwarder.crt", | |
"ssl key": "/usr/local/etc/logstash-forwarder/logstash-forwarder.key", | |
"ssl ca": "/usr/local/etc/logstash-forwarder/logstash-forwarder.crt", | |
"timeout": 15 | |
}, | |
"files": [ | |
{ | |
"paths": ["/usr/local/nginx/logs/access.log"], | |
"fields": { "type": "nginx-access" } | |
} | |
] | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# On Logstashserver | |
#wget -O https://download.elasticsearch.org/logstash/logstash/logstash-1.4.2.tar.gz | |
#tar zxvf logstash-1.4.2.tar.gz | |
#vi logstash-1.4.2/logstash.conf | |
input { | |
# * You'll need logstash-forwarder for use lumberjack in another computer - to centralize your logs in a server logstash | |
# it's necessary start a service to send info log | |
# cmd: ./logstash-forwarder -config forwarding.conf | |
lumberjack { | |
host => "localhost" | |
# The port to listen on | |
port => 5000 | |
# The paths to your ssl cert and key | |
ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt" | |
ssl_key => "/etc/pki/tls/private/logstash-forwarder.key" | |
# Set this to whatever you want. | |
type => "nginx-access" | |
} | |
} | |
filter { | |
# see nginx-access | |
if [type] == "nginx-access" { | |
grok { | |
match => { "message" => "%{NGINXACCESS}" } | |
} | |
} | |
} | |
output { | |
elasticsearch { | |
cluster => "logstash-dev" | |
protocol => http | |
node_name => "logstash" | |
} | |
stdout {} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# On Logstashserver | |
# vi ./logstash-1.4.2/patterns/nginx | |
NGUSERNAME [a-zA-Z\.\@\-\+_%]+ | |
NGUSER %{NGUSERNAME} | |
NGINXACCESS %{IPORHOST:clientip} %{NGUSER:ident} %{NGUSER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent} |
LIVINGSTONE:/usr/local/etc/logstash-forwarder
LIVINGSTONE:/usr/local/go/
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
start two services:
On logstashserver
On nginx server