Created
April 21, 2015 18:34
-
-
Save Hounddog/e347dc1c5b146c69c49a to your computer and use it in GitHub Desktop.
logstash debugging
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ubuntu@ip-10-0-0-23:~$ curl -XGET '10.0.0.215:9200' | |
| { | |
| "status" : 200, | |
| "name" : "Joey Bailey", | |
| "cluster_name" : "elasticsearch", | |
| "version" : { | |
| "number" : "1.4.4", | |
| "build_hash" : "c88f77ffc81301dfa9dfd81ca2232f09588bd512", | |
| "build_timestamp" : "2015-02-19T13:05:36Z", | |
| "build_snapshot" : false, | |
| "lucene_version" : "4.10.3" | |
| }, | |
| "tagline" : "You Know, for Search" | |
| } | |
| ubuntu@ip-10-0-0-23:~$ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ubuntu@ip-10-0-0-23:~$ ls -al /var/log/nginx/ | |
| total 48 | |
| drwxr-x--- 2 www-data adm 4096 Apr 21 17:39 . | |
| drwxrwxr-x 11 root syslog 4096 Apr 21 15:50 .. | |
| -rw-r--r-- 1 root root 0 Apr 21 17:39 access.log | |
| -rw-r--r-- 1 root root 36340 Apr 21 18:32 api.elephone.com-access.log | |
| -rw-r--r-- 1 root root 0 Apr 21 17:39 api.elephone.com-error.log | |
| -rw-r--r-- 1 root root 0 Apr 21 17:39 error.log |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| input { | |
| file { | |
| type => "nginx_access" | |
| path => "/var/log/nginx/{,api.}{,development.,staging.}elephone.*access.log" | |
| } | |
| file { | |
| type => "nginx_error" | |
| path => "/var/log/nginx/{,api.}{,development.,staging.}elephone.*error.log" | |
| } | |
| } | |
| filter { | |
| if [type] == "nginx_access" { | |
| grok { | |
| match => { "message" => "%{NGINXACCESS}" } | |
| } | |
| if [agent] == "Amazon Route 53 Health Check Service; ref:48063686-50df-47af-84af-21a22a997ff3; report http://amzn.to/1vsZADi" { | |
| drop {} | |
| } | |
| } | |
| if [type] == "nginx_error" { | |
| grok { | |
| match => { "message" => "%{NGINXERROR}" } | |
| } | |
| } | |
| date { | |
| match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ] | |
| } | |
| geoip { | |
| source => "client_ip" | |
| target => "geoip" | |
| } | |
| mutate { | |
| gsub => ["json_string", "\\x22", '"'] | |
| convert => [ "request_duration", "float" ] | |
| } | |
| if [json_string] != "-" { | |
| json { | |
| source => "json_string" | |
| } | |
| } | |
| } | |
| output { | |
| elasticsearch { | |
| host => "10.0.0.215" | |
| protocol => "http" | |
| } | |
| stdout { codec => rubydebug } | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ^Cubuntu@ip-10-0-0-23:~$ tail -f /var/log/nginx/api.elephone.com-access.log | |
| 10.0.0.23 10.0.0.103 [21/Apr/2015:18:33:54 +0000] "GET / HTTP/1.1" 200 1341 "-" "ELB-HealthChecker/1.0" "-" 0.456 0.456 | |
| 10.0.0.23 10.0.0.231 [21/Apr/2015:18:33:54 +0000] "GET / HTTP/1.1" 200 1341 "-" "ELB-HealthChecker/1.0" "-" 0.414 0.414 | |
| api.elephone.com 10.0.0.103 [21/Apr/2015:18:33:59 +0000] "GET / HTTP/1.1" 200 589 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/41.0.2272.76 Chrome/41.0.2272.76 Safari/537.36" "-" 0.384 0.384 | |
| api.elephone.com 10.0.0.103 [21/Apr/2015:18:34:00 +0000] "GET /zf-apigility-welcome/css/main.min.css HTTP/1.1" 404 534 "http://api.elephone.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/41.0.2272.76 Chrome/41.0.2272.76 Safari/537.36" "-" 0.760 0.760 | |
| api.elephone.com 10.0.0.103 [21/Apr/2015:18:34:00 +0000] "GET /zf-apigility/css/bootstrap.min.css HTTP/1.1" 200 99548 "http://api.elephone.com/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/41.0.2272.76 Chrome/41.0.2272.76 Safari/537.36" "-" 1.215 1.215 | |
| ubuntu@ip-10-0-0-23:~$ tail -f /var/log/logstash/logstash.log | |
| {:timestamp=>"2015-04-21T17:59:44.081000+0000", :message=>"Using milestone 2 input plugin 'file'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones", :level=>:warn} | |
| {:timestamp=>"2015-04-21T17:59:50.447000+0000", :message=>"Using milestone 2 filter plugin 'json'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones", :level=>:warn} | |
| {:timestamp=>"2015-04-21T18:04:18.237000+0000", :message=>"Using milestone 2 input plugin 'file'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones", :level=>:warn} | |
| {:timestamp=>"2015-04-21T18:04:23.445000+0000", :message=>"Using milestone 2 filter plugin 'json'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones", :level=>:warn} | |
| {:timestamp=>"2015-04-21T18:08:50.134000+0000", :message=>"Using milestone 2 input plugin 'file'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones", :level=>:warn} | |
| {:timestamp=>"2015-04-21T18:08:56.977000+0000", :message=>"Using milestone 2 filter plugin 'json'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones", :level=>:warn} | |
| {:timestamp=>"2015-04-21T18:15:20.938000+0000", :message=>"Using milestone 2 input plugin 'file'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones", :level=>:warn} | |
| {:timestamp=>"2015-04-21T18:15:29.146000+0000", :message=>"Using milestone 2 filter plugin 'json'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones", :level=>:warn} | |
| {:timestamp=>"2015-04-21T18:23:56.982000+0000", :message=>"Using milestone 2 input plugin 'file'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones", :level=>:warn} | |
| {:timestamp=>"2015-04-21T18:24:00.808000+0000", :message=>"Using milestone 2 filter plugin 'json'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones", :level=>:warn} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ubuntu@ip-10-0-0-23:~$ sudo /opt/logstash/bin/logstash -f /etc/logstash/conf.d/nginx.conf |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment