Skip to content

Instantly share code, notes, and snippets.

@jashmenn

jashmenn/.gdbinit

Created November 14, 2012 01:25
Show Gist options
  • Save jashmenn/4069637 to your computer and use it in GitHub Desktop.
Save jashmenn/4069637 to your computer and use it in GitHub Desktop.
Download ZIP
gdb ruby
Raw
.gdbinit
define session-ruby
source ~/.gdb/ruby
end
define uberrubydebug
set unwindonsignal on
thread apply all bt
session-ruby
redirect_stdout
eval "puts '=====================' + Process.pid.to_s"
eval "puts caller.join(\"\n\")"
restore_stdout
end
define ruby_stack_trace
set $VM_FRAME_MAGIC_METHOD = 0x11
set $VM_FRAME_MAGIC_BLOCK = 0x21
set $VM_FRAME_MAGIC_CLASS = 0x31
set $VM_FRAME_MAGIC_TOP = 0x41
set $VM_FRAME_MAGIC_FINISH = 0x51
set $VM_FRAME_MAGIC_CFUNC = 0x61
set $VM_FRAME_MAGIC_PROC = 0x71
set $VM_FRAME_MAGIC_IFUNC = 0x81
set $VM_FRAME_MAGIC_EVAL = 0x91
set $VM_FRAME_MAGIC_LAMBDA = 0xa1
set $VM_FRAME_MAGIC_MASK_BITS = 8
set $VM_FRAME_MAGIC_MASK = (~(~0 << $VM_FRAME_MAGIC_MASK_BITS))
set $RUBY_T_NODE = 0x1c
set $RUBY_T_MASK = 0x1f
set $T_NODE = $RUBY_T_NODE
set $T_MASK = $RUBY_T_MASK
set $FL_USHIFT = 12
set $FL_USER1 = ((1)<<($FL_USHIFT+1))
set $RSTRING_NOEMBED = $FL_USER1
set $p = th->cfp
while $p < th->stack + th->stack_size
set $VM_FRAME_TYPE = (($p)->flag & $VM_FRAME_MAGIC_MASK)
printf "#%-2d ", $p - th->cfp
set $nopos = 0
set $t = $VM_FRAME_TYPE
if $t == $VM_FRAME_MAGIC_METHOD
printf "METHOD "
end
if $t == $VM_FRAME_MAGIC_BLOCK
printf "BLOCK "
end
if $t == $VM_FRAME_MAGIC_CLASS
printf "CLASS "
end
if $t == $VM_FRAME_MAGIC_TOP
printf "TOP "
end
if $t == $VM_FRAME_MAGIC_FINISH
printf "FINISH "
set $nopos = 1
end
if $t == $VM_FRAME_MAGIC_CFUNC
printf "CFUNC "
end
if $t == $VM_FRAME_MAGIC_PROC
printf "PROC "
end
if $t == $VM_FRAME_MAGIC_IFUNC
printf "IFUNC "
end
if $t == $VM_FRAME_MAGIC_EVAL
printf "EVAL "
end
if $t == $VM_FRAME_MAGIC_LAMBDA
printf "LAMBDA "
end
if $t == 0
printf "------- "
end
if ! $nopos
if $p->iseq
if (((struct RBasic*)($p->iseq))->flags & $T_MASK) == $T_NODE
printf "<ifunc>"
else
if ((struct RBasic*)($p->iseq->filename))->flags & $RSTRING_NOEMBED
printf "%s:", (((struct RString*)($p->iseq->filename))->as).heap.ptr
else
printf "%s:", (((struct RString*)($p->iseq->filename))->as).ary
end
printf "%d %s", rb_vm_get_sourceline($p), rb_class2name($p->iseq->klass)
end
else
if $p->me->def->original_id
printf ":%s", rb_id2name($p->me->def->original_id)
end
end
end
printf "\n"
set $p = $p + 1
end
end
document ruby_stack_trace
Prints backtrace of all stack frames in the level of Ruby script.
Before invoking this command, a stack frame whose arguments include
a variable 'th' must be selected.
Notes: This command is intended for Ruby 1.9.3.
end
Raw
.gdbruby
# taken from http://eigenclass.org/hiki.rb?ruby+live+process+introspection
###################
# #### RUBY STUFF
define eval
call(rb_p(rb_eval_string_protect($arg0,(int*)0)))
end
define redirect_stdout
call rb_eval_string("$_old_stdout, $stdout = $stdout, File.open('/tmp/ruby-debug.pid', 'a'); $stdout.sync = true")
end
define restore_stdout
call rb_eval_string("$stdout = $_old_stdout")
end
define rb_finish
call rb_eval_string_protect("set_trace_func lambda{|event, file, line, id, binding, classname| if /return/ =~ event; sleep 0; set_trace_func(nil) end}",(int*)0)
tbreak rb_f_sleep
cont
end
define rb_object_counts
call rb_eval_string_protect("counts = Hash.new{|h,k| h[k] = 0}; ObjectSpace.each_object{|o| counts[o.class] += 1}; counts.sort_by{|k,c| c}.reverse.each{|k,c| puts '%7d %s' % [c, k] } ",(int*)0)
end
define rb_locals
eval "local_variables.map{|x| puts '%s = %s' % [x, eval(x)]}; nil"
end
define rp
if ($arg0 & 0x1)
echo T_FIXNUM\n
print (long)$arg0 >> 1
else
if ($arg0 == 0)
echo T_FALSE\n
else
if ($arg0 == 2)
echo T_TRUE\n
else
if ($arg0 == 4)
echo T_NIL\n
else
if ($arg0 == 6)
echo T_UNDEF\n
else
if (($arg0 & 0xff) == 0x0e)
echo T_SYMBOL\n
output $arg0 >> 8
echo \n
call rb_id2name($arg0 >> 8)
else
set $rbasic = (struct RBasic*)$arg0
# output $rbasic
# echo \ =\
# output *$rbasic
# echo \n
set $flags = (*$rbasic).flags & 0x3f
if ($flags == 0x01)
echo T_NIL\n
echo impossible\n
else
if ($flags == 0x02)
echo T_OBJECT\n
print *(struct RObject*)$rbasic
else
if ($flags == 0x03)
echo T_CLASS\n
# rb_classname($arg0)
else
if ($flags == 0x04)
echo T_ICLASS\n
print *(struct RClass*)$rbasic
else
if ($flags == 0x05)
echo T_MODULE\n
print *(struct RClass*)$rbasic
else
if ($flags == 0x06)
echo T_FLOAT\n
print *(struct RFloat*)$rbasic
else
if ($flags == 0x07)
echo T_STRING\n
print *(struct RString*)$rbasic
else
if ($flags == 0x08)
echo T_REGEXP\n
print *(struct RRegexp*)$rbasic
else
if ($flags == 0x09)
echo T_ARRAY\n
print *(struct RArray*)$rbasic
else
if ($flags == 0x0a)
echo T_FIXNUM\n
echo impossible\n
else
if ($flags == 0x0b)
echo T_HASH\n
print *(struct RHash*)$rbasic
else
if ($flags == 0x0c)
echo T_STRUCT\n
print *(struct RStruct*)$rbasic
else
if ($flags == 0x0d)
echo T_BIGNUM\n
print *(struct RBignum*)$rbasic
else
if ($flags == 0x0e)
echo T_FILE\n
print *(struct RFile*)$rbasic
else
if ($flags == 0x20)
echo T_TRUE\n
echo impossible\n
else
if ($flags == 0x21)
echo T_FALSE\n
echo impossible\n
else
if ($flags == 0x22)
echo T_DATA\n
print *(struct RData*)$rbasic
else
if ($flags == 0x23)
echo T_MATCH\n
print *(struct RMatch*)$rbasic
else
if ($flags == 0x24)
echo T_SYMBOL\n
echo impossible\n
else
if ($flags == 0x3c)
echo T_UNDEF\n
echo impossible\n
else
if ($flags == 0x3d)
echo T_VARMAP\n
else
if ($flags == 0x3e)
echo T_SCOPE\n
else
if ($flags == 0x3f)
echo T_NODE\n
print (NODE*)$arg0
else
echo Unknown\n
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
document rp
ruby¤ÎÁȤ߹þ¤ß¥ª¥Ö¥¸¥§¥¯¥È¤òɽ¼¨¤¹¤ë
end
define nd_type
print nodetype($arg0)
end
document nd_type
ruby node ¤Î·¿¤òɽ¼¨
end
define nd_file
print ((NODE*)$arg0)->nd_file
end
document nd_file
node ¤Î¥½¡¼¥¹¥Õ¥¡¥¤¥ë̾¤òɽ¼¨
end
define nd_line
print nodeline($arg0)
end
document nd_line
node ¤Î¹ÔÈÖ¹æ¤òɽ¼¨
end
# ruby node ¤Î¥á¥ó¥Ð¤òɽ¼¨
define nd_head
print "u1.node"
what $arg0.u1.node
p $arg0.u1.node
end
define nd_alen
print "u2.argc"
what $arg0.u2.argc
p $arg0.u2.argc
end
define nd_next
print "u3.node"
what $arg0.u3.node
p $arg0.u3.node
end
define nd_cond
print "u1.node"
what $arg0.u1.node
p $arg0.u1.node
end
define nd_body
print "u2.node"
what $arg0.u2.node
p $arg0.u2.node
end
define nd_else
print "u3.node"
what $arg0.u3.node
p $arg0.u3.node
end
define nd_orig
print "u3.value"
what $arg0.u3.value
p $arg0.u3.value
end
define nd_resq
print "u2.node"
what $arg0.u2.node
p $arg0.u2.node
end
define nd_ensr
print "u3.node"
what $arg0.u3.node
p $arg0.u3.node
end
define nd_1st
print "u1.node"
what $arg0.u1.node
p $arg0.u1.node
end
define nd_2nd
print "u2.node"
what $arg0.u2.node
p $arg0.u2.node
end
define nd_stts
print "u1.node"
what $arg0.u1.node
p $arg0.u1.node
end
define nd_entry
print "u3.entry"
what $arg0.u3.entry
p $arg0.u3.entry
end
define nd_vid
print "u1.id"
what $arg0.u1.id
p $arg0.u1.id
end
define nd_cflag
print "u2.id"
what $arg0.u2.id
p $arg0.u2.id
end
define nd_cval
print "u3.value"
what $arg0.u3.value
p $arg0.u3.value
end
define nd_cnt
print "u3.cnt"
what $arg0.u3.cnt
p $arg0.u3.cnt
end
define nd_tbl
print "u1.tbl"
what $arg0.u1.tbl
p $arg0.u1.tbl
end
define nd_var
print "u1.node"
what $arg0.u1.node
p $arg0.u1.node
end
define nd_ibdy
print "u2.node"
what $arg0.u2.node
p $arg0.u2.node
end
define nd_iter
print "u3.node"
what $arg0.u3.node
p $arg0.u3.node
end
define nd_value
print "u2.node"
what $arg0.u2.node
p $arg0.u2.node
end
define nd_aid
print "u3.id"
what $arg0.u3.id
p $arg0.u3.id
end
define nd_lit
print "u1.value"
what $arg0.u1.value
p $arg0.u1.value
end
define nd_frml
print "u1.node"
what $arg0.u1.node
p $arg0.u1.node
end
define nd_rest
print "u2.argc"
what $arg0.u2.argc
p $arg0.u2.argc
end
define nd_opt
print "u1.node"
what $arg0.u1.node
p $arg0.u1.node
end
define nd_recv
print "u1.node"
what $arg0.u1.node
p $arg0.u1.node
end
define nd_mid
print "u2.id"
what $arg0.u2.id
p $arg0.u2.id
end
define nd_args
print "u3.node"
what $arg0.u3.node
p $arg0.u3.node
end
define nd_noex
print "u1.id"
what $arg0.u1.id
p $arg0.u1.id
end
define nd_defn
print "u3.node"
what $arg0.u3.node
p $arg0.u3.node
end
define nd_old
print "u1.id"
what $arg0.u1.id
p $arg0.u1.id
end
define nd_new
print "u2.id"
what $arg0.u2.id
p $arg0.u2.id
end
define nd_cfnc
print "u1.cfunc"
what $arg0.u1.cfunc
p $arg0.u1.cfunc
end
define nd_argc
print "u2.argc"
what $arg0.u2.argc
p $arg0.u2.argc
end
define nd_cname
print "u1.id"
what $arg0.u1.id
p $arg0.u1.id
end
define nd_super
print "u3.node"
what $arg0.u3.node
p $arg0.u3.node
end
define nd_modl
print "u1.id"
what $arg0.u1.id
p $arg0.u1.id
end
define nd_clss
print "u1.value"
what $arg0.u1.value
p $arg0.u1.value
end
define nd_beg
print "u1.node"
what $arg0.u1.node
p $arg0.u1.node
end
define nd_end
print "u2.node"
what $arg0.u2.node
p $arg0.u2.node
end
define nd_state
print "u3.state"
what $arg0.u3.state
p $arg0.u3.state
end
define nd_rval
print "u2.value"
what $arg0.u2.value
p $arg0.u2.value
end
define nd_nth
print "u2.argc"
what $arg0.u2.argc
p $arg0.u2.argc
end
define nd_tag
print "u1.id"
what $arg0.u1.id
p $arg0.u1.id
end
define nd_tval
print "u2.value"
what $arg0.u2.value
p $arg0.u2.value
end
define rb_p
call rb_p($arg0)
end
define rb_id2name
call rb_id2name($arg0)
end
define rb_classname
call classname($arg0)
rb_p $
p *(struct RClass*)$arg0
end
define rb_backtrace
call rb_backtrace()
end
Raw
commands
gdb -batch -quiet -ex "uberrubydebug" /usr/bin/ruby19 -f `ps aux | grep mini_queue | grep -v grep | awk '{print $2}' | rl -c 1`
Raw
gistfile1.txt
Script level stack trace for Ruby 1.9.1 with gdb
pagediscussionview sourcehistory
Contents [hide]
1 Introduction
2 .gdbinit
3 Instructions
3.1 Start gdb with a target process id
3.2 Set a width widely to avoid line feed in the middle of line
3.3 Determine what threads are there
3.4 Choose a target thread in question
3.5 Print a back trace
3.6 Find the innermost frame with an argument 'th='
3.7 Print a stack trace in the level of Ruby script
4 Another means using an embedded debug function
Introduction
This gdb macro prints script level stack trace for Ruby and would help developers find a cause of problem with a running Ruby process.
This macro has been developed based on the functions control_frame_dump() and rb_vmdebug_stack_dump_raw() in the source file vm_dump.c of Ruby 1.9.1.
(gdb) ruby_stack_trace
#0 CFUNC :recvfrom
#1 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/xxx.rb:70 UDPSocket
#2 BLOCK /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/xxx.rb:124 Object
#3 METHOD <internal:prelude>:8 Mutex
#4 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/xxx.rb:115 Foo::Bar::Xxx
#5 BLOCK /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/yyy.rb:31 Object
#6 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/zzz.rb:286 Foo::Zzz::UDP::Multicast
#7 FINISH
#8 CFUNC :new
#9 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/yyy.rb:26 Foo::Bar::Yyy
#10 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/yyy.rb:95 Foo::Yyy
#11 BLOCK /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/yyy.rb:54 Object
#12 FINISH
#13 TOP
.gdbinit
To utilize it, copy the following lines to your $HOME/.gdbinit
define ruby_stack_trace
set $VM_FRAME_MAGIC_METHOD = 0x11
set $VM_FRAME_MAGIC_BLOCK = 0x21
set $VM_FRAME_MAGIC_CLASS = 0x31
set $VM_FRAME_MAGIC_TOP = 0x41
set $VM_FRAME_MAGIC_FINISH = 0x51
set $VM_FRAME_MAGIC_CFUNC = 0x61
set $VM_FRAME_MAGIC_PROC = 0x71
set $VM_FRAME_MAGIC_IFUNC = 0x81
set $VM_FRAME_MAGIC_EVAL = 0x91
set $VM_FRAME_MAGIC_LAMBDA = 0xa1
set $VM_FRAME_MAGIC_MASK_BITS = 8
set $VM_FRAME_MAGIC_MASK = (~(~0 << $VM_FRAME_MAGIC_MASK_BITS))
set $RUBY_T_NODE = 0x1c
set $RUBY_T_MASK = 0x1f
set $T_NODE = $RUBY_T_NODE
set $T_MASK = $RUBY_T_MASK
set $FL_USHIFT = 12
set $FL_USER1 = ((1)<<($FL_USHIFT+1))
set $RSTRING_NOEMBED = $FL_USER1
set $p = th->cfp
while $p < th->stack + th->stack_size
set $VM_FRAME_TYPE = (($p)->flag & $VM_FRAME_MAGIC_MASK)
printf "#%-2d ", $p - th->cfp
set $nopos = 0
set $t = $VM_FRAME_TYPE
if $t == $VM_FRAME_MAGIC_METHOD
printf "METHOD "
end
if $t == $VM_FRAME_MAGIC_BLOCK
printf "BLOCK "
end
if $t == $VM_FRAME_MAGIC_CLASS
printf "CLASS "
end
if $t == $VM_FRAME_MAGIC_TOP
printf "TOP "
end
if $t == $VM_FRAME_MAGIC_FINISH
printf "FINISH "
set $nopos = 1
end
if $t == $VM_FRAME_MAGIC_CFUNC
printf "CFUNC "
end
if $t == $VM_FRAME_MAGIC_PROC
printf "PROC "
end
if $t == $VM_FRAME_MAGIC_IFUNC
printf "IFUNC "
end
if $t == $VM_FRAME_MAGIC_EVAL
printf "EVAL "
end
if $t == $VM_FRAME_MAGIC_LAMBDA
printf "LAMBDA "
end
if $t == 0
printf "------- "
end
if ! $nopos
if $p->iseq
if (((struct RBasic*)($p->iseq))->flags & $T_MASK) == $T_NODE
printf "<ifunc>"
else
if ((struct RBasic*)($p->iseq->filename))->flags & $RSTRING_NOEMBED
printf "%s:", (((struct RString*)($p->iseq->filename))->as).heap.ptr
else
printf "%s:", (((struct RString*)($p->iseq->filename))->as).ary
end
printf "%d %s", rb_vm_get_sourceline($p), rb_class2name($p->iseq->klass)
end
else
if $p->method_id
printf ":%s", rb_id2name($p->method_id)
end
end
end
printf "\n"
set $p = $p + 1
end
end
document ruby_stack_trace
Prints backtrace of all stack frames in the level of Ruby script.
Before invoking this command, a stack frame whose arguments include
a variable 'th' must be selected.
Notes: This command is intended for Ruby 1.9.1.
end
Instructions
Start gdb with a target process id
# gdb /usr/local/bin/ruby 19289
GNU gdb (GDB) Red Hat Enterprise Linux (7.0.1-23.el5)
Copyright (C) 2009 Free Software Foundation, Inc.
.....
Set a width widely to avoid line feed in the middle of line
(gdb) set width 999
Determine what threads are there
(gdb) info threads
8 Thread 0x41d8f940 (LWP 19292) 0x0000003e45c0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
7 Thread 0x405eb940 (LWP 19293) 0x0000003e45c0aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
6 Thread 0x40a2f940 (LWP 19294) 0x0000003e45c0aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
5 Thread 0x41e7b940 (LWP 19295) 0x0000003e450caf36 in poll () from /lib64/libc.so.6
4 Thread 0x40985940 (LWP 19296) 0x0000003e45c0aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
3 Thread 0x41653940 (LWP 19297) 0x0000003e45c0aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
2 Thread 0x40bf2940 (LWP 19298) 0x0000003e45c0d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0
* 1 Thread 0x2b5cc5772050 (LWP 19289) 0x0000003e45c0d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0
Choose a target thread in question
(gdb) thread 5
[Switching to thread 5 (Thread 0x41e7b940 (LWP 19295))]#0 0x0000003e450caf36 in poll () from /lib64/libc.so.6
Print a back trace
(gdb) bt
#0 0x0000003e450caf36 in poll () from /lib64/libc.so.6
#1 0x0000003e4f208e00 in __libc_res_nsend () from /lib64/libresolv.so.2
#2 0x0000003e4f207c26 in __libc_res_nquery () from /lib64/libresolv.so.2
#3 0x00002aaaad25f0b5 in _nss_dns_gethostbyaddr2_r () from /lib64/libnss_dns.so.2
#4 0x00002aaaad25f835 in _nss_dns_gethostbyaddr_r () from /lib64/libnss_dns.so.2
#5 0x0000003e450e9875 in gethostbyaddr_r@@GLIBC_2.2.5 () from /lib64/libc.so.6
#6 0x0000003e450f1ef6 in getnameinfo () from /lib64/libc.so.6
#7 0x00002aaaabec9a3b in ipaddr (sockaddr=0x41e79f04, norevlookup=0) at socket.c:1021
#8 0x00002aaaabecd0d7 in s_recvfrom (sock=<value optimized out>, argc=<value optimized out>, argv=<value optimized out>, from=RECV_IP) at socket.c:665
#9 0x00000000004cb000 in vm_call_cfunc (th=0xda7f500, reg_cfp=0x2aaaacd4db30, num=1, id=24936, oid=21352, recv=228929344, klass=226346904, flag=8, mn=0xd7dc568, blockptr=0x0) at vm_insnhelper.c:384
#10 0x00000000004cc020 in vm_call_method (th=0xda7f500, cfp=0x2aaaacd4db30, num=1, blockptr=0x0, flag=8, id=24936, mn=0xda52b18, recv=228929344) at vm_insnhelper.c:517
#11 0x00000000004ce770 in vm_exec_core (th=0xda7f500, initial=<value optimized out>) at insns.def:999
#12 0x00000000004d21ca in vm_exec (th=0xda7f500) at vm.c:1080
#13 0x00000000004d444b in vm_call0 (th=0xda7f500, klass=<value optimized out>, recv=223833792, id=456, oid=456, argc=4, argv=0x2aaaacc4e078, body=0xd4124a8, nosuper=0) at vm_eval.c:57
#14 0x00000000004cb9ba in rb_call0 (recv=223833792, mid=456, argc=4, argv=0x2aaaacc4e078) at vm_eval.c:249
#15 rb_call (recv=223833792, mid=456, argc=4, argv=0x2aaaacc4e078) at vm_eval.c:255
#16 rb_funcall2 (recv=223833792, mid=456, argc=4, argv=0x2aaaacc4e078) at vm_eval.c:433
#17 0x000000000043ca12 in rb_class_new_instance (argc=4, argv=0x2aaaacc4e078, klass=<value optimized out>) at object.c:1482
#18 0x00000000004cb000 in vm_call_cfunc (th=0xda7f500, reg_cfp=0x2aaaacd4de30, num=4, id=960, oid=960, recv=222373928, klass=217839048, flag=0, mn=0xcfb98d0, blockptr=0x2aaaacd4de58) at vm_insnhelper.c:384
#19 0x00000000004cc020 in vm_call_method (th=0xda7f500, cfp=0x2aaaacd4de30, num=4, blockptr=0x2aaaacd4de58, flag=0, id=960, mn=0xcfb9898, recv=222373928) at vm_insnhelper.c:517
#20 0x00000000004ce770 in vm_exec_core (th=0xda7f500, initial=<value optimized out>) at insns.def:999
#21 0x00000000004d21ca in vm_exec (th=0xda7f500) at vm.c:1080
#22 0x00000000004d27bc in invoke_block_from_c (th=0xda7f500, block=0xda7f870, self=228927720, argc=0, argv=0x0, blockptr=0x0, cref=0x0) at vm.c:524
#23 0x00000000004d2c72 in rb_vm_invoke_proc (th=0xda7f500, proc=0xda7f870, self=228927720, argc=0, argv=0xda94b10, blockptr=0x0) at vm.c:571
#24 0x00000000004dc6ce in thread_start_func_2 (th=0xda7f500, stack_start=0x41e7b130) at thread.c:391
#25 0x00000000004dc7fe in thread_start_func_1 (th_ptr=0x41e78400) at thread_pthread.c:351
#26 0x0000003e45c0673d in start_thread () from /lib64/libpthread.so.0
#27 0x0000003e450d3d1d in clone () from /lib64/libc.so.6
Find the innermost frame with an argument 'th='
(gdb) frame 9
#9 0x00000000004cb000 in vm_call_cfunc (th=0xda7f500, reg_cfp=0x2aaaacd4db30, num=1, id=24936, oid=21352, recv=228929344, klass=226346904, flag=8, mn=0xd7dc568, blockptr=0x0) at vm_insnhelper.c:384
384 vm_insnhelper.c: No such file or directory.
in vm_insnhelper.c
Print a stack trace in the level of Ruby script
(gdb) ruby_stack_trace
#0 CFUNC :recvfrom
#1 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/xxx.rb:70 UDPSocket
#2 BLOCK /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/xxx.rb:124 Object
#3 METHOD <internal:prelude>:8 Mutex
#4 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/xxx.rb:115 Foo::Bar::Xxx
#5 BLOCK /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/yyy.rb:31 Object
#6 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/zzz.rb:286 Foo::Zzz::UDP::Multicast
#7 FINISH
#8 CFUNC :new
#9 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/yyy.rb:26 Foo::Bar::Yyy
#10 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/yyy.rb:95 Foo::Yyy
#11 BLOCK /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/yyy.rb:54 Object
#12 FINISH
#13 TOP
Another means using an embedded debug function
There is a function in the source file of Ruby 1.9.1, which prints a stack trace.
(gdb) print rb_vmdebug_stack_dump_raw(th, th.cfp)
$1 = void
If nothing is shown, determine where both 'stdout' and 'stderr' are directed to and look for the outputs in those files.
# lsof -p 999 -a -d 1,2
COMMAND PID USER FD TYPE DEVICE SIZE NODE NAME
foo 999 user 1w REG 253,3 1767757910 6416259 /var/log/foo.log
foo 999 user 2w REG 253,3 1767757910 6416259 /var/log/foo.log
Alternatively, reopen both 'stdout' and 'stderr' for the current terminal.
(gdb) shell tty
/dev/pts/6
(gdb) print freopen("/dev/pts/6", "w", stdout)
$2 = 1161107328
(gdb) print freopen("/dev/pts/6", "w", stderr)
$3 = 1161107552
Do it again.
(gdb) print rb_vmdebug_stack_dump_raw(th, th.cfp)
-- control frame ----------
c:0010 p:---- s:0032 b:0032 l:000031 d:000031 CFUNC :select
c:0009 p:0038 s:0025 b:0025 l:000019 d:000024 BLOCK /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/server.rb:360
c:0008 p:0019 s:0023 b:0023 l:000022 d:000022 METHOD <internal:prelude>:8
c:0007 p:0012 s:0020 b:0020 l:000019 d:000019 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/server.rb:358
c:0006 p:0049 s:0016 b:0016 l:000015 d:000015 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/server.rb:330
c:0005 p:0012 s:0012 b:0012 l:000011 d:000011 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/console_workers.rb:27
c:0004 p:0030 s:0009 b:0009 l:0016d8 d:0016d8 METHOD /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/workers.rb:95
c:0003 p:0009 s:0006 b:0006 l:000348 d:000005 BLOCK /usr/local/lib/ruby/gems/1.9.1/gems/foo-0.0.1/lib/foo/workers.rb:54
c:0002 p:---- s:0004 b:0004 l:000003 d:000003 FINISH
c:0001 p:---- s:0002 b:0002 l:000001 d:000001 TOP
---------------------------
$4 = void
Navigation
Main Page
All pages
Categories
Recent changes
Random page
Help
Search
Toolbox
What links here
Related changes
Special pages
Printable version
Permanent link
log in
This page was last modified on 11 February 2011, at 09:40. This page has been accessed 786 times. Privacy policy About ToraWiki Disclaimers
Powered by MediaWiki 1.15.1
Raw
instructions.txt
# http://projects.puppetlabs.com/projects/1/wiki/Puppet_Introspection
Puppet Introspection
Okay. If you have an actively flakey puppet master, here are the steps you can follow to attach to it with GDB and do some introspection (in Ruby, no less!)
Install GDB
Follow whatever steps make sense for your platform. Something like:
$ yum install gdb
Or:
$ apt-get install gdb
Populate your .gdbinit and .gdb directories
In \~/.gdbinit:
define session-ruby
source ~/.gdb/ruby
end
Next, you need to download the live process file for Ruby and place it in \~/.gdb/ruby:
$ mkdir -p ~/.gdb
$ wget 'http://eigenclass.org/hiki.rb?c=plugin;plugin=attach_download;p=ruby+live+process+introspection;file_name=ruby'
$ mv ruby ~/.gdb
Attach to the process
Attach to the puppetmaster process, load the ruby helpers, and redirect stdout:
$ ps ax | grep puppetmasterd
21170 ttyp5 S+ 0:02 /usr/bin/ruby /srv/ruby/1.8.6/bin/puppetmasterd --no-daemonize --debug --trace
The important part here is the very first segment of the line (the PID):
$ gdb /usr/bin/ruby 21170
The first argument to gdb should be the full path to the ruby executable that is running your puppetmasterd.
Start your debug session
From above, you should see a bunch of output like this:
...
Loaded symbols for /srv/ruby/1.8.6/lib/ruby/1.8/i686-linux/nkf.so
0xbfffe402 in +kernel_vsyscall ()
And then you’ll wind up at:
(gdb)
First things first, load up the ruby helpers:
(gdb) session-ruby
Then, redirect stdout to a file, so we can capture the output for later:
(gdb) redirect_stdout
Open up a second terminal, and you can watch the output from your debug session:
$ tail -f /tmp/ruby-debug.PID
Where PID above is equal to the PID you attached to with gdb.
Now that you are watching, flip back to your gdb session and…
Get some data
Now you can retrieve some data:
(gdb) eval "caller"
(gdb) rb_object_counts
(gdb) eval "total = \[\[ObjectSpace\]\].each_object(Array)\{\|x\| puts '---'; p x \}; puts \\"---\\nTotal Arrays: \#{total}\\""
Further Information
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment