Created
January 21, 2013 17:46
-
-
Save pcn/4587793 to your computer and use it in GitHub Desktop.
loggly OOM freeze when there is an output 503
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2012-10-05T03:21:11.60448 W, [2012-10-05T03:21:11.604000 #2204] WARN -- : Output thread exception {"timestamp":"2012-10-05T03:21:11.604000 +0000", | |
| "plugin":"LogStash::Outputs::Loggly: {\"host\"=>\"logs01.loggly.net\", \"key\"=>\"SANITIZED\", \"type\"=>\"\", \"tags\"=>[], \"proto\"=>\"http\"}","exception":"503 \"Server down\"","backtrace":["file:/opt/knewton/logstash/logstash-monolithic.jar!/META-INF/jruby.home/ | |
| lib/ruby/1.9/net/http.rb:2303:in `error!'","file:/opt/knewton/logstash/logstash-monolithic.jar!/logstash/outputs/loggly.rb:77:in `receive'","file:/ | |
| opt/knewton/logstash/logstash-monolithic.jar!/logstash/outputs/base.rb:48:in `handle'","file:/opt/knewton/logstash/logstash-monolithic.jar!/logstas | |
| h/agent.rb:674:in `run_output'","file:/opt/knewton/logstash/logstash-monolithic.jar!/logstash/agent.rb:333:in `start_output'"],"message":"Output th | |
| read exception","level":"warn"} | |
| 2012-10-05T09:28:03.30075 Exception in thread "filter|worker|0" java.lang.OutOfMemoryError: Java heap space | |
| 2012-10-05T09:28:03.30114 at org.jruby.util.ByteList.<init>(ByteList.java:91) | |
| 2012-10-05T09:28:03.30122 at org.jruby.RubyString.addByteLists(RubyString.java:1058) | |
| 2012-10-05T09:28:03.30129 at org.jruby.RubyString.op_plus19(RubyString.java:1048)2012-10-05T09:28:03.30136 at org.jruby.RubyString$i$1$0$op_plus19.call(RubyString$i$1$0$op_plus19.gen:65535) | |
| 2012-10-05T09:28:03.30144 at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:167)2012-10-05T09:28:03.30151 at org.jruby.javasupport.util.RuntimeHelpers.opAsgnWithMethod(RuntimeHelpers.java:1610) | |
| 2012-10-05T09:28:03.30158 at rubyjit.append_061A6576C9A152011A83A5EEEA2DCBCBFB395CA5.__file__(file:/opt/knewton/logstash/logstash-monolithic. | |
| jar!/logstash/event.rb:146) | |
| 2012-10-05T09:28:03.30166 at rubyjit.append_061A6576C9A152011A83A5EEEA2DCBCBFB395CA5.__file__(file:/opt/knewton/logstash/logstash-monolithic. | |
| jar!/logstash/event.rb) | |
| 2012-10-05T09:28:03.30173 at org.jruby.internal.runtime.methods.JittedMethod.call(JittedMethod.java:167) | |
| 2012-10-05T09:28:03.30180 at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:167)2012-10-05T09:28:03.30188 at rubyjit.filter_0A0B2EF6C098195D17BA2FDFAC1C9A3CB030698E.__file__(file:/opt/knewton/logstash/logstash-monolithic. | |
| jar!/logstash/filters/multiline.rb:130)2012-10-05T09:28:03.30195 at rubyjit.filter_0A0B2EF6C098195D17BA2FDFAC1C9A3CB030698E.__file__(file:/opt/knewton/logstash/logstash-monolithic. | |
| jar!/logstash/filters/multiline.rb)2012-10-05T09:28:03.30202 at org.jruby.internal.runtime.methods.JittedMethod.call(JittedMethod.java:187)2012-10-05T09:28:03.30209 at org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:322) | |
| 2012-10-05T09:28:03.30216 at org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:178)2012-10-05T09:28:03.30224 at org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:187)2012-10-05T09:28:03.30231 at rubyjit.filter_3F595277C3CDE099D0342BAFC73B86C737D96873.block_1$RUBY$__file__(file:/opt/knewton/logstash/logstas | |
| h-monolithic.jar!/logstash/filterworker.rb:57)2012-10-05T09:28:03.30239 at rubyjit$filter_3F595277C3CDE099D0342BAFC73B86C737D96873$block_1$RUBY$__file__.call(rubyjit$filter_3F595277C3CDE099D0342BAFC73B86C737D96873$block_1$RUBY$__file__:65535) | |
| 2012-10-05T09:28:03.30246 at org.jruby.runtime.CompiledBlock19.yield(CompiledBlock19.java:130)2012-10-05T09:28:03.30253 at org.jruby.runtime.Block.yield(Block.java:130)2012-10-05T09:28:03.30262 at org.jruby.RubyArray.eachCommon(RubyArray.java:1605) | |
| 2012-10-05T09:28:03.30269 at org.jruby.RubyArray.each(RubyArray.java:1612)2012-10-05T09:28:03.30276 at org.jruby.RubyArray$i$0$0$each.call(RubyArray$i$0$0$each.gen:65535)2012-10-05T09:28:03.30284 at org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:142) | |
| 2012-10-05T09:28:03.30291 at org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:153) | |
| 2012-10-05T09:28:03.30298 at rubyjit.filter_3F595277C3CDE099D0342BAFC73B86C737D96873.block_0$RUBY$__file__(file:/opt/knewton/logstash/logstash-monolithic.jar!/logstash/filterworker.rb:49) | |
| 2012-10-05T09:28:03.30305 at rubyjit$filter_3F595277C3CDE099D0342BAFC73B86C737D96873$block_0$RUBY$__file__.call(rubyjit$filter_3F595277C3CDE099D0342BAFC73B86C737D96873$block_0$RUBY$__file__:65535) | |
| 2012-10-05T09:28:03.30313 at org.jruby.runtime.CompiledBlock19.yield(CompiledBlock19.java:130) | |
| 2012-10-05T09:28:03.30320 at org.jruby.runtime.Block.yield(Block.java:130) | |
| 2012-10-05T09:28:03.30327 at org.jruby.RubyArray.eachCommon(RubyArray.java:1605) | |
| 2012-10-05T09:28:03.30334 at org.jruby.RubyArray.each(RubyArray.java:1612) | |
| 2012-10-05T09:28:03.30341 at org.jruby.RubyArray$i$0$0$each.call(RubyArray$i$0$0$each.gen:65535) | |
| 2013-01-21T16:55:42.78481 W, [2013-01-21T16:55:42.784000 #2204] WARN -- : SIGTERM received, shutting down. {"timestamp":"2013-01-21T16:55:42.784000 +0000","message":"SIGTERM received, shutting down.","level":"warn"} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment