stefanvangastel · March 19, 2017 09:30 · Jan 6, 2017 · Jan 6, 2017 · Jan 4, 2017
diff --git a/logstash_my_app.conf b/logstash_my_app.conf
@@ -4,11 +4,11 @@ input {
 
   tcp {
     codec => multiline {
-               # Merge lines based on an exception
-               pattern => "\[%{GREEDYDATA}\]"
-               negate => "true"
-               what => "previous"
-            }
+       # Merge lines based on an exception
+       pattern => "\[%{GREEDYDATA}\]"
+       negate => "true"
+       what => "previous"
+    }
     port => 5140
     type => "syslog"
   }

diff --git a/logstash_my_app.conf b/logstash_my_app.conf
@@ -3,12 +3,12 @@
 input {
 
   tcp {
-   codec => multiline {
-        # Merge lines based on an exception
-        pattern => "\[%{GREEDYDATA}\]"
-        negate => "true"
-        what => "previous"
-    }
+    codec => multiline {
+               # Merge lines based on an exception
+               pattern => "\[%{GREEDYDATA}\]"
+               negate => "true"
+               what => "previous"
+            }
     port => 5140
     type => "syslog"
   }

diff --git a/bootstrap.php b/bootstrap.php
@@ -0,0 +1,8 @@
+# Change default log engine at the end of bootstrap.php
+# See https://book.cakephp.org/3.0/en/core-libraries/logging.html#logging-to-syslog
+Log::config('default', [
+    'engine' => 'Syslog',
+    'flag' => LOG_ODELAY | LOG_PERROR,
+    'facility' => LOG_LOCAL7, // Use local7 as dedicated syslog facilty for this app (https://en.wikipedia.org/wiki/Syslog#Facility)
+    'prefix' => 'MY_APP'
+]);
diff --git a/elasticsearch_result.json b/elasticsearch_result.json
@@ -0,0 +1,30 @@
+{
+  "_index": "logstash-2017.01.04",
+  "_type": "syslog",
+  "_id": "AVlp9VLfjrqyulk7dql7",
+  "_score": 1,
+  "_source": {
+    "@timestamp": "2017-01-04T14:50:01.685Z",
+    "message": "<187>Jan  4 15:50:00 ubuntu MY_APP: error: [Cake\\Network\\Exception\\InternalErrorException] No user account found in header or REMOTE_USER var. Is proxy forwarding working ok?\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: Request URL: /some-url\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: Stack Trace:\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: #0 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(697): Example\\LdapAuthenticate\\Auth\\LdapAuthenticate->getUser(Object(Cake\\Network\\Request))\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: #1 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(292): Cake\\Controller\\Component\\AuthComponent->_getUser()\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: #2 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(258): Cake\\Controller\\Component\\AuthComponent->authCheck(Object(Cake\\Event\\Event))\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: #3 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(385): Cake\\Controller\\Component\\AuthComponent->startup(Object(Cake\\Event\\Event))\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: #4 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(355): Cake\\Event\\EventManager->_callListener(Array, Object(Cake\\Event\\Event))\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: #5 /var/www/vendor/cakephp/cakephp/src/Event/EventDispatcherTrait.php(78): Cake\\Event\\EventManager->dispatch(Object(Cake\\Event\\Event))\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: #6 /var/www/vendor/cakephp/cakephp/src/Controller/Controller.php(495): Cake\\Controller\\Controller->dispatchEvent('Controller.star...')\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: #7 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(109): Cake\\Controller\\Controller->startupProcess()\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: #8 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(87): Cake\\Routing\\Dispatcher->_invoke(Object(App\\Controller\\AppController))\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: #9 /var/www/webroot/index.php(37): Cake\\Routing\\Dispatcher->dispatch(Object(Cake\\Network\\Request), Object(Cake\\Network\\Response))\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: #10 {main}\n<187>Jan  4 15:50:00 ubuntu MY_APP: error: \n<187>Jan  4 15:38:10 ubuntu MY_APP: error: ",
+    "@version": "1",
+    "tags": [
+      "multiline",
+      "cakephp_log"
+    ],
+    "host": "192.168.0.99",
+    "port": 60677,
+    "type": "syslog",
+    "timestamp": "Jan  4 15:50:00",
+    "logsource": "ubuntu",
+    "program": "MY_APP",
+    "loglevel": "error",
+    "exception": "Cake\\Network\\Exception\\InternalErrorException",
+    "path": "/some-url",
+    "stacktrace": "\n#0 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(697): Example\\LdapAuthenticate\\Auth\\LdapAuthenticate->getUser(Object(Cake\\Network\\Request))\n#1 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(292): Cake\\Controller\\Component\\AuthComponent->_getUser()\n#2 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(258): Cake\\Controller\\Component\\AuthComponent->authCheck(Object(Cake\\Event\\Event))\n#3 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(385): Cake\\Controller\\Component\\AuthComponent->startup(Object(Cake\\Event\\Event))\n#4 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(355): Cake\\Event\\EventManager->_callListener(Array, Object(Cake\\Event\\Event))\n#5 /var/www/vendor/cakephp/cakephp/src/Event/EventDispatcherTrait.php(78): Cake\\Event\\EventManager->dispatch(Object(Cake\\Event\\Event))\n#6 /var/www/vendor/cakephp/cakephp/src/Controller/Controller.php(495): Cake\\Controller\\Controller->dispatchEvent('Controller.star...')\n#7 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(109): Cake\\Controller\\Controller->startupProcess()\n#8 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(87): Cake\\Routing\\Dispatcher->_invoke(Object(App\\Controller\\AppController))\n#9 /var/www/webroot/index.php(37): Cake\\Routing\\Dispatcher->dispatch(Object(Cake\\Network\\Request), Object(Cake\\Network\\Response))\n#10 {main}\n\n"
+  },
+  "fields": {
+    "@timestamp": [
+      1483541401685
+    ]
+  }
+}
diff --git a/logstash_my_app.conf b/logstash_my_app.conf
@@ -0,0 +1,46 @@
+# File: /etc/logstash/conf.d/my_app.conf
+
+input {
+
+  tcp {
+   codec => multiline {
+        # Merge lines based on an exception
+        pattern => "\[%{GREEDYDATA}\]"
+        negate => "true"
+        what => "previous"
+    }
+    port => 5140
+    type => "syslog"
+  }
+
+}
+
+filter{
+
+    grok{
+        match => {"message"=>"%{SYSLOGBASE2} %{LOGLEVEL:loglevel}: \[%{DATA:exception}\] %{GREEDYDATA} Request URL: %{URIPATHPARAM:path}%{GREEDYDATA}Stack Trace:%{GREEDYDATA:stacktrace}"}
+        overwrite =>["message"]
+        add_tag => ["cakephp_log"]
+   }
+    mutate {
+        # Remove the prefixed syslog base (e.g.: 'Jan 4 15:39:35 ubuntu my_app: error:')
+        gsub => ["stacktrace","...\ ..\ ..\:..\:..\ .*\ .*\:\ error: ",""]
+
+        # Remove the syslog prefix (e.g.: '<187>')
+        gsub => ["stacktrace","<[0-9]+>",""]
+    }
+
+}
+
+output {
+
+    # Send to Elasticsearch
+    elasticsearch {
+        hosts => "es.mydomain.com"
+        index => "logstash-%{+YYYY.MM.dd}"
+    }
+
+    # Debug
+    stdout { codec => rubydebug }
+
+}
diff --git a/my_app.log b/my_app.log
@@ -0,0 +1,15 @@
+FILE: /var/log/my_app.log
+Jan  4 15:50:01 ubuntu MY_APP: error: [Cake\Network\Exception\InternalErrorException] Holy cow, something went wrong!
+Jan  4 15:50:01 ubuntu MY_APP: error: Request URL: /some-url
+Jan  4 15:50:01 ubuntu MY_APP: error: Stack Trace:
+Jan  4 15:50:01 ubuntu MY_APP: error: #0 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(697): Example\LdapAuthenticate\Auth\LdapAuthenticate->getUser(Object(Cake\Network\Request))
+Jan  4 15:50:01 ubuntu MY_APP: error: #1 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(292): Cake\Controller\Component\AuthComponent->_getUser()
+Jan  4 15:50:01 ubuntu MY_APP: error: #2 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(258): Cake\Controller\Component\AuthComponent->authCheck(Object(Cake\Event\Event))
+Jan  4 15:50:01 ubuntu MY_APP: error: #3 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(385): Cake\Controller\Component\AuthComponent->startup(Object(Cake\Event\Event))
+Jan  4 15:50:01 ubuntu MY_APP: error: #4 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(355): Cake\Event\EventManager->_callListener(Array, Object(Cake\Event\Event))
+Jan  4 15:50:01 ubuntu MY_APP: error: #5 /var/www/vendor/cakephp/cakephp/src/Event/EventDispatcherTrait.php(78): Cake\Event\EventManager->dispatch(Object(Cake\Event\Event))
+Jan  4 15:50:01 ubuntu MY_APP: error: #6 /var/www/vendor/cakephp/cakephp/src/Controller/Controller.php(495): Cake\Controller\Controller->dispatchEvent('Controller.star...')
+Jan  4 15:50:01 ubuntu MY_APP: error: #7 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(109): Cake\Controller\Controller->startupProcess()
+Jan  4 15:50:01 ubuntu MY_APP: error: #8 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(87): Cake\Routing\Dispatcher->_invoke(Object(App\Controller\AppController))
+Jan  4 15:50:01 ubuntu MY_APP: error: #9 /var/www/webroot/index.php(37): Cake\Routing\Dispatcher->dispatch(Object(Cake\Network\Request), Object(Cake\Network\Response))
+Jan  4 15:50:01 ubuntu MY_APP: error: #10 {main}
diff --git a/rsyslog_my_app.conf b/rsyslog_my_app.conf
@@ -0,0 +1,4 @@
+# File: /etc/rsyslog.d/my_app.conf
+# Use local7 as log facility for my_app, send the logs to both a Logstash server over TCP (@@) and a local file in /var/log
+local7.*    @@logstash.mydomain.com:5140
+local7.*    /var/log/my_app.log