mranney

> process.versions
{ node: '0.6.16-pre',
  v8: '3.6.6.25',
  ares: '1.7.5-DEV',
  uv: '0.6',
  openssl: '0.9.8q' }

mranney

Uncaught exception: AssertionError: false == true
    at CleartextStream._pull (tls.js:397:3)
    at SecurePair.cycle (tls.js:727:20)
    at EncryptedStream.write (tls.js:130:13)
    at Socket.ondata (stream.js:38:26)
    at Socket.emit (events.js:67:17)
    at TCP.onread (net.js:367:14)

mranney

diff --git a/lib/http.js b/lib/http.js
index f7109d7..6cc5473 100644
--- a/lib/http.js
+++ b/lib/http.js
@@ -1127,6 +1127,8 @@ ClientRequest.prototype.onSocket = function(socket) {
 
     var freeParser = function() {
       if (parser) {
+        // next line added by mjr to track down mystery parser exception
+        parser.socket.ondata = parser.socket.onend = null;

mranney

[root@prod-2119 ~]# dtrace -n 'syscall::*brk*:entry /pid == 6714 || pid == 4918 || pid == 6474 || pid == 5497 || pid == 6972 || pid == 5254 || pid == 4713 || pid == 6230/ { @[jstack(100, 8000)] = count(); }'
dtrace: description 'syscall::*brk*:entry ' matched 1 probe
^C


              libc.so.1`_brk_unlocked+0x15
              libc.so.1`sbrk+0x35
              libc.so.1`_morecore+0xfc
              libc.so.1`_malloc_unlocked+0x17f
              libc.so.1`malloc+0x35

mranney

              libc.so.1`_brk_unlocked+0x15
              libc.so.1`sbrk+0x35
              libc.so.1`_morecore+0x2e
              libc.so.1`_malloc_unlocked+0x17f
              libc.so.1`malloc+0x35
              libstdc++.so.6.0.16`_Znwj+0x29
              libstdc++.so.6.0.16`_Znaj+0x1d
              node`_ZN4node6Buffer7ReplaceEPcjPFvS1_PvES2_+0x118
              node`_ZN4node6BufferC1EN2v86HandleINS1_6ObjectEEEj+0xb0
              node`_ZN4node6Buffer3NewERKN2v89ArgumentsE+0x8e

mranney

(gdb) info registers
eax            0x0	0
ecx            0x0	0
edx            0x0	0
ebx            0x0	0
esp            0x0	0x0
ebp            0x98dab500	0x98dab500
esi            0xffffff00	-256
edi            0xb8bacc40	-1195717568
eip            0x0	0

mranney

[root@prod-2187 ~]# prstat -mLcp `pgrep -d, -f router_node.js` 2
Please wait...
   PID USERNAME USR SYS TRP TFL DFL LCK SLP LAT VCX ICX SCL SIG PROCESS/LWPID 
 18927 root      47 2.9 0.1 0.0 0.0 0.0  50 0.4 263 170  5K   0 node/1
 18867 root      46 2.9 0.1 0.0 0.0 0.0  51 0.4 260 148  5K   0 node/1
 18747 root      46 3.0 0.1 0.0 0.0 0.0  51 0.4 275 148  5K   0 node/1
 18627 root      41 2.9 0.1 0.0 0.0 0.0  56 0.5 278 153  5K   0 node/1
 18687 root      40 2.8 0.1 0.0 0.0 0.0  56 0.5 272 153  5K   0 node/1
 18508 root      38 2.9 0.0 0.0 0.0 0.0  59 0.4 283 127  5K   0 node/1
 18568 root      37 2.9 0.0 0.0 0.0 0.0  60 0.4 288 125  5K   0 node/1

mranney

process="prod-nr681" loglevel="error" timestamp="1331885153823" hostname="prod-2178" Uncaught exception: Error: accept ECONNABORTED
    at errnoException (net.js:646:11)
    at TCP.onconnection (net.js:831:24)
Uncaught exception: Error: accept ECONNABORTED
    at errnoException (net.js:646:11)
    at TCP.onconnection (net.js:831:24)
Trace: 
    at EventEmitter.<anonymous> (/voxer/deploy/server/router_node.js:258:13)
    at EventEmitter.emit (events.js:88:20)

mranney

mjr@prod-04:~$ telnet prod-17-internal 8091 
Trying 10.28.60.232...
Connected to prod-17-internal.voxer.com.
Escape character is '^]'.
GET /riak/mjrtest/2 HTTP/1.1
Host: foo
Accept: */*

HTTP/1.1 404 Object Not Found
Server: MochiWeb/1.1 WebMachine/1.9.0 (someone had painted it blue)

mranney

CPU     ID                    FUNCTION:NAME
  7   9350                        :tick-10s 


              node`_ZN2v88internalL28AllocateFixedArrayWithFillerEPNS0_4HeapEiNS0_13PretenureFlagEPNS0_6ObjectE+0x53
              node`_ZN2v88internal7JSArray10InitializeEi+0x84
              node`_ZN2v88internal7Factory10NewJSArrayEiNS0_13PretenureFlagE+0x7f
              node`_ZN2v85Array3NewEi+0xbe
              node`_ZN4node6Parser19on_headers_completeEP11http_parser+0x1bf
              node`http_parser_execute+0xff8