gistfile1.txt

# rbd info rbd-images/360 --debug-rbd=20 --debug-ms=1
2017-06-06 09:08:02.553941 7ffba03ef000  1  Processor -- start
2017-06-06 09:08:02.554092 7ffba03ef000  1 -- - start start
2017-06-06 09:08:02.554475 7ffba03ef000  1 -- - --> 10.0.0.10:6789/0 -- auth(proto 0 30 bytes epoch 0) v1 -- 0x55a973f9f140 con 0
2017-06-06 09:08:02.554662 7ffb89bdb700  1 -- 10.0.0.10:0/999315689 learned_addr learned my addr 10.0.0.10:0/999315689
2017-06-06 09:08:02.555152 7ffb7bfff700  1 -- 10.0.0.10:0/999315689 <== mon.1 10.0.0.10:6789/0 1 ==== mon_map magic: 0 v1 ==== 418+0+0 (2163738294 0 0) 0x7ffb7c002120 con 0x55a973fa5860
2017-06-06 09:08:02.555218 7ffb7bfff700  1 -- 10.0.0.10:0/999315689 <== mon.1 10.0.0.10:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (2872788003 0 0) 0x7ffb7c002610 con 0x55a973fa5860
2017-06-06 09:08:02.555307 7ffb7bfff700  1 -- 10.0.0.10:0/999315689 --> 10.0.0.10:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- 0x7ffb70001780 con 0
2017-06-06 09:08:02.555775 7ffb7bfff700  1 -- 10.0.0.10:0/999315689 <== mon.1 10.0.0.10:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (900415387 0 0) 0x7ffb7c001010 con 0x55a973fa5860
2017-06-06 09:08:02.555872 7ffb7bfff700  1 -- 10.0.0.10:0/999315689 --> 10.0.0.10:6789/0 -- auth(proto 2 165 bytes epoch 0) v1 -- 0x7ffb70001ff0 con 0
2017-06-06 09:08:02.556423 7ffb7bfff700  1 -- 10.0.0.10:0/999315689 <== mon.1 10.0.0.10:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 564+0+0 (1641177789 0 0) 0x7ffb7c002940 con 0x55a973fa5860
2017-06-06 09:08:02.556503 7ffb7bfff700  1 -- 10.0.0.10:0/999315689 --> 10.0.0.10:6789/0 -- mon_subscribe({monmap=0+}) v2 -- 0x55a973fa3730 con 0
2017-06-06 09:08:02.556564 7ffba03ef000  1 -- 10.0.0.10:0/999315689 --> 10.0.0.10:6789/0 -- mon_subscribe({mgrmap=0+}) v2 -- 0x55a973fa3730 con 0
2017-06-06 09:08:02.556683 7ffba03ef000  1 -- 10.0.0.10:0/999315689 --> 10.0.0.10:6789/0 -- mon_subscribe({osdmap=0}) v2 -- 0x55a973fa3c10 con 0
2017-06-06 09:08:02.556707 7ffb7bfff700  1 -- 10.0.0.10:0/999315689 <== mon.1 10.0.0.10:6789/0 5 ==== mon_map magic: 0 v1 ==== 418+0+0 (2163738294 0 0) 0x7ffb7c001410 con 0x55a973fa5860
2017-06-06 09:08:02.556799 7ffb7bfff700  1 -- 10.0.0.10:0/999315689 <== mon.1 10.0.0.10:6789/0 6 ==== mgrmap(e 175) v1 ==== 138+0+0 (3339592447 0 0) 0x7ffb7c001200 con 0x55a973fa5860
2017-06-06 09:08:02.556902 7ffb7bfff700  1 -- 10.0.0.10:0/999315689 <== mon.1 10.0.0.10:6789/0 7 ==== osd_map(23968..23968 src has 23396..23968) v3 ==== 36892+0+0 (95933669 0 0) 0x7ffb7c001650 con 0x55a973fa5860
2017-06-06 09:08:02.557583 7ffba03ef000  5 librbd::AioImageRequestWQ: 0x55a973faa4b0 : ictx=0x55a973fa9130
2017-06-06 09:08:02.557691 7ffba03ef000 20 librbd::ImageState: 0x55a973f9f900 open
2017-06-06 09:08:02.557701 7ffba03ef000 10 librbd::ImageState: 0x55a973f9f900 0x55a973f9f900 send_open_unlock
2017-06-06 09:08:02.557712 7ffba03ef000 10 librbd::image::OpenRequest: 0x55a973faae00 send_v2_detect_header
2017-06-06 09:08:02.557816 7ffba03ef000  1 -- 10.0.0.10:0/999315689 --> 10.0.0.11:6820/2445 -- osd_op(unknown.0.0:1 1.e856ba99 rbd_id.360 [stat] snapc 0=[] ack+read+known_if_redirected e23968) v7 -- 0x55a973fb0560 con 0
2017-06-06 09:08:02.558594 7ffb8a3dc700  1 -- 10.0.0.10:0/999315689 <== osd.91 10.0.0.11:6820/2445 1 ==== osd_op_reply(1 rbd_id.360 [stat] v0'0 uv5281 ondisk = 0) v7 ==== 130+0+16 (1903503687 0 272051827) 0x7ffb84001fa0 con 0x55a973fabe40
2017-06-06 09:08:02.558654 7ffb78ff9700 10 librbd::image::OpenRequest: handle_v2_detect_header: r=0
2017-06-06 09:08:02.558668 7ffb78ff9700 10 librbd::image::OpenRequest: 0x55a973faae00 send_v2_get_id
2017-06-06 09:08:02.558706 7ffb78ff9700  1 -- 10.0.0.10:0/999315689 --> 10.0.0.11:6820/2445 -- osd_op(unknown.0.0:2 1.e856ba99 rbd_id.360 [call rbd.get_id] snapc 0=[] ack+read+known_if_redirected e23968) v7 -- 0x7ffb60002200 con 0
2017-06-06 09:08:02.559094 7ffb8a3dc700  1 -- 10.0.0.10:0/999315689 <== osd.91 10.0.0.11:6820/2445 2 ==== osd_op_reply(2 rbd_id.360 [call] v0'0 uv5281 ondisk = 0) v7 ==== 130+0+16 (1874511254 0 4211329641) 0x7ffb840024e0 con 0x55a973fabe40
2017-06-06 09:08:02.559120 7ffb78ff9700 10 librbd::image::OpenRequest: handle_v2_get_id: r=0
2017-06-06 09:08:02.559132 7ffb78ff9700 10 librbd::image::OpenRequest: 0x55a973faae00 send_v2_get_immutable_metadata
2017-06-06 09:08:02.559194 7ffb78ff9700  1 -- 10.0.0.10:0/999315689 --> 10.0.0.12:6801/13088 -- osd_op(unknown.0.0:3 1.d94bc0e1 rbd_header.8c503b52115c [call rbd.get_size,call rbd.get_object_prefix] snapc 0=[] ack+read+known_if_redirected e23968) v7 -- 0x7ffb60009850 con 0
2017-06-06 09:08:02.560084 7ffb89bdb700  1 -- 10.0.0.10:0/999315689 <== osd.62 10.0.0.12:6801/13088 1 ==== osd_op_reply(3 rbd_header.8c503b52115c [call,call] v0'0 uv0 ack = -2 ((2) No such file or directory)) v7 ==== 185+0+0 (3529719043 0 0) 0x7ffb7c002b50 con 0x7ffb60005220
2017-06-06 09:08:02.560144 7ffb78ff9700 10 librbd::image::OpenRequest: handle_v2_get_immutable_metadata: r=-2
2017-06-06 09:08:02.560155 7ffb78ff9700 -1 librbd::image::OpenRequest: failed to retreive immutable metadata: (2) No such file or directory
2017-06-06 09:08:02.560181 7ffb78ff9700 10 librbd::image::OpenRequest: 0x55a973faae00 send_close_image
2017-06-06 09:08:02.560191 7ffb78ff9700 10 librbd::image::CloseRequest: 0x7ffb60001c50 send_shut_down_update_watchers
2017-06-06 09:08:02.560193 7ffb78ff9700 20 librbd::ImageState: 0x55a973f9f900 shut_down_update_watchers
2017-06-06 09:08:02.560194 7ffb78ff9700 20 librbd::ImageState: 0x55a973fa9bc0 ImageUpdateWatchers::shut_down
2017-06-06 09:08:02.560197 7ffb78ff9700 20 librbd::ImageState: 0x55a973fa9bc0 ImageUpdateWatchers::shut_down: completing shut down
2017-06-06 09:08:02.560223 7ffb6ffff700 10 librbd::image::CloseRequest: 0x7ffb60001c50 handle_shut_down_update_watchers: r=0
2017-06-06 09:08:02.560226 7ffb6ffff700 10 librbd::image::CloseRequest: 0x7ffb60001c50 send_shut_down_aio_queue
2017-06-06 09:08:02.560231 7ffb6ffff700  5 librbd::AioImageRequestWQ: shut_down: in_flight=0
2017-06-06 09:08:02.560246 7ffb6ffff700 10 librbd::image::CloseRequest: 0x7ffb60001c50 handle_shut_down_aio_queue: r=0
2017-06-06 09:08:02.560250 7ffb6ffff700 10 librbd::image::CloseRequest: 0x7ffb60001c50 send_flush
2017-06-06 09:08:02.560254 7ffb6ffff700 10 librbd::image::CloseRequest: 0x7ffb60001c50 handle_flush: r=0
2017-06-06 09:08:02.560256 7ffb6ffff700 10 librbd::image::CloseRequest: 0x7ffb60001c50 send_flush_readahead
rbd: error opening image 360: (2) No such file or directory
2017-06-06 09:08:02.560259 7ffb6ffff700 10 librbd::image::CloseRequest: 0x7ffb60001c50 handle_flush_readahead: r=0
2017-06-06 09:08:02.560260 7ffb6ffff700 10 librbd::image::CloseRequest: 0x7ffb60001c50 send_shut_down_cache
2017-06-06 09:08:02.560262 7ffb6ffff700 10 librbd::image::CloseRequest: 0x7ffb60001c50 handle_shut_down_cache: r=0
2017-06-06 09:08:02.560264 7ffb6ffff700 10 librbd::image::CloseRequest: 0x7ffb60001c50 send_flush_op_work_queue
2017-06-06 09:08:02.560267 7ffb6ffff700 10 librbd::image::CloseRequest: 0x7ffb60001c50 handle_flush_op_work_queue: r=0
2017-06-06 09:08:02.560269 7ffb6ffff700 10 librbd::image::OpenRequest: handle_close_image: r=0
2017-06-06 09:08:02.560278 7ffb6ffff700 10 librbd::ImageState: 0x55a973f9f900 0x55a973f9f900 handle_open: r=-2
2017-06-06 09:08:02.560469 7ffba03ef000  1 -- 10.0.0.10:0/999315689 >> 10.0.0.12:6801/13088 conn(0x7ffb60005220 :-1 s=STATE_OPEN pgs=275 cs=1 l=1).mark_down
2017-06-06 09:08:02.560519 7ffba03ef000  1 -- 10.0.0.10:0/999315689 >> 10.0.0.11:6820/2445 conn(0x55a973fabe40 :-1 s=STATE_OPEN pgs=299 cs=1 l=1).mark_down
2017-06-06 09:08:02.560694 7ffba03ef000  1 -- 10.0.0.10:0/999315689 >> 10.0.0.10:6789/0 conn(0x55a973fa5860 :-1 s=STATE_OPEN pgs=20989 cs=1 l=1).mark_down
2017-06-06 09:08:02.560821 7ffba03ef000  1 -- 10.0.0.10:0/999315689 shutdown_connections
2017-06-06 09:08:02.560913 7ffb7bfff700  1 -- 10.0.0.10:0/999315689 >> 10.184.52.17:6813/791 conn(0x7ffb70009ae0 :-1 s=STATE_CLOSED pgs=212468653 cs=1 l=0).mark_down
2017-06-06 09:08:02.561133 7ffba03ef000  1 -- 10.0.0.10:0/999315689 shutdown_connections
2017-06-06 09:08:02.561235 7ffba03ef000  1 -- 10.0.0.10:0/999315689 wait complete.
2017-06-06 09:08:02.561249 7ffba03ef000  1 -- 10.0.0.10:0/999315689 >> 10.0.0.10:0/999315689 conn(0x55a973f91d90 :-1 s=STATE_NONE pgs=0 cs=0 l=0).mark_down