messages

Sep 16 13:10:26 node5 kernel: [29355.892378]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.892384]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.892389]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.892394]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.892399]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.892404]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.892408]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.892413]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.892418]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.892422]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.892427]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.892431] cron          D ffff882db7c9cd70     0 170696 124643 1564 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.892436]  ffff882d62511d18 0000000000000082 ffff882d62511ce0 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.892443]  0000000000000000 00000000000c2a11 ffff88012be10770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.892449]  ffff88012be10760 0000000101ba3fcb ffff882db7c9d338 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.892456] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.892461]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.892466]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.892471]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.892476]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.892481]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.892485]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.892490]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.892495]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.892499]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.892504]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.892508] cron          D ffff882de260cdb0     0 170697 111383 1288 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.892513]  ffff88376b89fd18 0000000000000086 ffff88376b89fd68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.892520]  ffff882de260cdb0 0000000000000000 ffffffff8100beae ffff88376b89fd18
Sep 16 13:10:26 node5 kernel: [29355.892527]  0000000000000246 0000000000000000 ffff882de260d378 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.892533] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.892537]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.892542]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.892547]  [<ffffffff810559be>] ? mutex_spin_on_owner+0x5e/0xc0
Sep 16 13:10:26 node5 kernel: [29355.892553]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.892558]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.892562]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.892567]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.892572]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.892577]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.892582]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.892586]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.892591]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.892596]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.892600] cron          D ffff88169a5e6780     0 170698  43503  952 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.892605]  ffff88178cfff758 0000000000000082 0000000000000002 0000000000000202
Sep 16 13:10:26 node5 kernel: [29355.892611]  ffff88178cfff788 ffffffffa02f9942 000c00220000177b 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.892618]  ffff88178cfff728 ffffffff00000000 ffff88169a5e6d48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.892624] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.892653]  [<ffffffffa02f9942>] ? dmu_zfetch+0x4b2/0xe30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.892663]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.892670]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.892693]  [<ffffffffa02d75fa>] arc_buf_add_ref+0x9a/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.892717]  [<ffffffffa02e1946>] __dbuf_hold_impl+0x66/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.892741]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.892765]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.892790]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.892818]  [<ffffffffa02fa8d4>] ? dnode_rele+0x54/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.892844]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.892879]  [<ffffffffa034754f>] zap_get_leaf_byblk+0x4f/0x2c0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.892914]  [<ffffffffa034782a>] zap_deref_leaf+0x6a/0x80 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.892948]  [<ffffffffa0347f20>] fzap_lookup+0x60/0x120 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.892983]  [<ffffffffa034dad1>] zap_lookup_norm+0xe1/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893017]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893050]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893084]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893118]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893125]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.893159]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893194]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893199]  [<ffffffff811bab82>] __lookup_hash+0x102/0x160
Sep 16 13:10:26 node5 kernel: [29355.893204]  [<ffffffff811bb50a>] lookup_hash+0x3a/0x50
Sep 16 13:10:26 node5 kernel: [29355.893208]  [<ffffffff811c0182>] do_filp_open+0x2e2/0xc50
Sep 16 13:10:26 node5 kernel: [29355.893214]  [<ffffffff8116f236>] ? find_vma+0x46/0x80
Sep 16 13:10:26 node5 kernel: [29355.893219]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.893224]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.893228]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.893233]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.893238]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.893244]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.893250]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.893253] cron          D ffff883a1256cdf0     0 170699 844465 2977 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.893258]  ffff883a0a2f3d18 0000000000000086 ffff883a0a2f3ce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.893265]  ffff883a1256cdf0 000000000088a770 ffff88012be90770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.893272]  ffff88012be90760 0000000101ba3fe2 ffff883a1256d3b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.893279] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.893283]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.893289]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.893295]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.893299]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.893305]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.893310]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.893314]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.893319]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.893324]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.893329]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.893334]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.893337] cron          D ffff883a1256c300     0 170700 844465 2977 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.893342]  ffff8839c75cdd18 0000000000000086 ffff8839c75cdce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.893349]  ffff883a1256c300 000000000033539a ffff882100c10770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.893356]  ffff882100c10760 0000000101ba3fe0 ffff883a1256c8c8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.893362] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.893366]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.893371]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.893377]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.893382]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.893387]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.893392]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.893397]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.893401]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.893406]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.893411]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.893415]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.893420]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.893424] cron          D ffff88361543a2c0     0 170701 334896 1193 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.893429]  ffff8839ced8dab8 0000000000000082 ffff8839ced8da80 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.893436]  0000000000000282 000000000088a770 ffff88012be90770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.893443]  ffff88012be90760 0000000101ba3fe6 ffff88361543a888 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.893449] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.893453]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.893463]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.893470]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.893493]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893518]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893543]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893567]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893593]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893623]  [<ffffffffa02ea8e5>] dmu_buf_hold_array+0x65/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893648]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893684]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893689]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.893724]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893729]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.893734]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.893740]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.893743] cron          D ffff882931908eb0     0 170702 112332 1291 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.893749]  ffff883bfe8657a8 0000000000000086 0000000000010287 0000000000000202
Sep 16 13:10:26 node5 kernel: [29355.893755]  ffff883bfe8657d8 ffffffffa02f97bb 0000000000000004 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.893762]  ffff883bfe865778 ffffffff00000000 ffff882931909478 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.893769] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.893796]  [<ffffffffa02f97bb>] ? dmu_zfetch+0x32b/0xe30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893806]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.893814]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.893837]  [<ffffffffa02d75fa>] arc_buf_add_ref+0x9a/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893861]  [<ffffffffa02e1946>] __dbuf_hold_impl+0x66/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893885]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893909]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893935]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893961]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.893995]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.894000]  [<ffffffff81194d1f>] ? cache_alloc_refill+0x9f/0x230
Sep 16 13:10:26 node5 kernel: [29355.894013]  [<ffffffffa020027a>] ? kmem_alloc_debug+0x13a/0x4c0 [spl]
Sep 16 13:10:26 node5 kernel: [29355.894047]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.894080]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.894113]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.894146]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.894179]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.894185]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.894219]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.894232]  [<ffffffffa020aae3>] ? crhold+0x13/0x20 [spl]
Sep 16 13:10:26 node5 kernel: [29355.894266]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.894271]  [<ffffffff811bab82>] __lookup_hash+0x102/0x160
Sep 16 13:10:26 node5 kernel: [29355.894276]  [<ffffffff811bb50a>] lookup_hash+0x3a/0x50
Sep 16 13:10:26 node5 kernel: [29355.894280]  [<ffffffff811c0182>] do_filp_open+0x2e2/0xc50
Sep 16 13:10:26 node5 kernel: [29355.894286]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.894291]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.894296]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.894301]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.894306]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.894310]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.894316]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.894319] cron          D ffff882db7c9c280     0 170703  47915 1127 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.894325]  ffff883a17d5dd18 0000000000000086 ffff883a17d5dce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.894331]  ffff882db7c9c280 0000000000749ad5 ffff882100d10770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.894338]  ffff882100d10760 0000000101ba4017 ffff882db7c9c848 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.894345] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.894349]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.894355]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.894361]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.894365]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.894371]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.894376]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.894380]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.894385]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.894390]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.894394]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.894399]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.894403] cron          D ffff883d128a2c30     0 170704 120659 1179 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.894408]  ffff8839a4b8fd18 0000000000000082 0000000000000018 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.894415]  ffff883d128a2c30 0000000000000000 0000000000000028 ffff883a0dabb4a8
Sep 16 13:10:26 node5 kernel: [29355.894422]  ffff883a0dfe8c60 ffff883d128a2c30 ffff883d128a31f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.894428] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.894433]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.894438]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.894444]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.894448]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.894453]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.894458]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.894463]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.894468]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.894472]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.894477]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.894482]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.894486] cron          D ffff8839730cac30     0 170705 120659 1179 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.894491]  ffff883e5251fd18 0000000000000086 ffff883e5251fd68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.894498]  ffff8839730cac30 0000000000000000 0000000000000028 ffff88378c0c29e0
Sep 16 13:10:26 node5 kernel: [29355.894504]  ffff88388ee1ec60 ffff8839730cac30 ffff8839730cb1f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.894511] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.894515]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.894521]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.894526]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.894531]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.894536]  [<ffffffff8116f250>] ? find_vma+0x60/0x80
Sep 16 13:10:26 node5 kernel: [29355.894540]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.894545]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.894550]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.894555]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.894559]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.894564]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.894569]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.894573] cron          D ffff883936b99030     0 170706  25731  736 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.894578]  ffff883792aafd18 0000000000000082 0000000000000000 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.894585]  ffff883936b99030 00000000007f3353 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.894591]  0000000000000000 0000000101ba4041 ffff883936b995f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.894598] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.894602]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.894608]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.894613]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.894617]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.894622]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.894627]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.894632]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.894637]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.894641]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.894646]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.894651]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.894655] cron          D ffff8837ad68ce30     0 170707 158472 2502 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.894660]  ffff883791edfd18 0000000000000082 ffff883791edfd68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.894667]  ffff8837ad68ce30 0000000000000000 0000000000000028 ffff88378f2cc468
Sep 16 13:10:26 node5 kernel: [29355.894673]  ffff8817be2acae8 ffff8837ad68ce30 ffff8837ad68d3f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.894680] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.894684]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.894690]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.894695]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.894699]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.894704]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.894709]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.894714]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.894719]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.894723]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.894728]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.894733]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.894737] cron          D ffff883908f260c0     0 170708 279251 2165 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.894742]  ffff883791f9fd18 0000000000000082 ffff883791f9fd68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.894749]  ffff883908f260c0 0000000000000000 0000000000000028 ffff8836c00417c8
Sep 16 13:10:26 node5 kernel: [29355.894755]  ffff882e21f40be8 ffff883908f260c0 ffff883908f26688 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.894762] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.894766]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.894772]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.894777]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.894782]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.894787]  [<ffffffff811bb33c>] ? getname_flags+0x1fc/0x260
Sep 16 13:10:26 node5 kernel: [29355.894791]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.894796]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.894801]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.894805]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.894810]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.894814] cron          D ffff88381ca19070     0 170709 279251 2165 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.894819]  ffff883790be3d18 0000000000000082 ffff883790be3d68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.894826]  ffff883f0a90ec80 0000000000000000 0000000000000028 ffff883766dd9210
Sep 16 13:10:26 node5 kernel: [29355.894832]  ffff883766dd8be8 ffff88381ca19070 ffff88381ca19638 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.894839] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.894843]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.894849]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.894854]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.894858]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.894863]  [<ffffffff810a7148>] ? up_read+0x18/0x30
Sep 16 13:10:26 node5 kernel: [29355.894868]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.894873]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.894877]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.894882]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.894887]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.894891]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.894896]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.894900] cron          D ffff8813cf1b43c0     0 170710 107782 1214 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.894905]  ffff881f9a687d18 0000000000000086 ffff881f9a687ce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.894912]  ffff8813cf1b43c0 00000000009b177c ffff882100c10770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.894919]  ffff882100c10760 0000000101ba4062 ffff8813cf1b4988 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.894925] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.894929]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.894935]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.894940]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.894945]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.894950]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.894955]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.894960]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.894965]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.894969]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.894974]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.894979]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.894983] cron          D ffff881858cd6400     0 170711 107782 1214 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.894988]  ffff88175fcd3d18 0000000000000082 ffff88175fcd3ce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.894994]  ffff881858cd6400 00000000010028e2 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.895001]  ffff8820737b0800 0000000101ba4069 ffff881858cd69c8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.895008] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.895012]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.895018]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.895023]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.895028]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.895033]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.895038]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.895042]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.895047]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.895052]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.895056]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.895061]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.895065] cron          D ffff8837ad68c340     0 170712  57166 1168 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.895070]  ffff883790667a98 0000000000000086 ffff883790666000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.895077]  0000000000000000 0000000000000002 ffff882073672c70 000200da00000000
Sep 16 13:10:26 node5 kernel: [29355.895084]  0000000000000282 0000000000000030 ffff8837ad68c908 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.895090] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.895099]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.895106]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.895130]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895155]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895180]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895203]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895232]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895261]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895287]  [<ffffffffa02ea8b9>] dmu_buf_hold_array+0x39/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895295]  [<ffffffffa002a9c8>] ? avl_add+0x38/0x50 [zavl]
Sep 16 13:10:26 node5 kernel: [29355.895321]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895356]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895361]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.895395]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895401]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.895406]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.895411]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.895415] cron          D ffff8807304dd3b0     0 170713 106959 1227 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.895421]  ffff88022b451a98 0000000000000082 ffff88022b450000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.895427]  0000000000000000 0000000000000002 ffff884073428bf0 000200da00000000
Sep 16 13:10:26 node5 kernel: [29355.895434]  0000000000000286 0000000000000030 ffff8807304dd978 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.895441] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.895450]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.895457]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.895481]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895505]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895530]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895554]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895582]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895587]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.895616]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895642]  [<ffffffffa02ea8b9>] dmu_buf_hold_array+0x39/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895668]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895703]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895708]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.895743]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.895748]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.895753]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.895758]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.895762] cron          D ffff8818cef84380     0 170714 115299  243 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.895767]  ffff88042beffd18 0000000000000082 ffff88042beffce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.895774]  ffff8818cef84380 00000000015be4ff ffff8840734c2410 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.895781]  ffff8840734c2400 0000000101ba40c6 ffff8818cef84948 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.895787] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.895791]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.895798]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.895803]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.895808]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.895813]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.895818]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.895823]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.895828]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.895833]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.895837]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.895843]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.895846] cron          D ffff88022ecd6eb0     0 170715 115299  243 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.895851]  ffff88022b455d18 0000000000000086 ffff88022b455d68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.895858]  ffff88022ecd6eb0 0000000000000000 0000000000000028 ffff88022c8dadc8
Sep 16 13:10:26 node5 kernel: [29355.895865]  ffff88316661d390 ffff88022ecd6eb0 ffff88022ecd7478 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.895871] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.895875]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.895882]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.895887]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.895891]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.895897]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.895902]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.895906]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.895911]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.895916]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.895920]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.895925]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.895929] cron          D ffff88178cf8cef0     0 170716  99549 1223 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.895934]  ffff882020205d18 0000000000000086 ffffffff8100beae ffff882020205d18
Sep 16 13:10:26 node5 kernel: [29355.895941]  0000000000000246 0000000000000000 ffff883b1aa24780 ffff883d1bd9f6e8
Sep 16 13:10:26 node5 kernel: [29355.895947]  0000000000000114 ffff882100ddec80 ffff88178cf8d4b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.895954] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.895958]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.895964]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.895969]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.895974]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.895979]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.895984]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.895989]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.895993]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.895998]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.896003]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.896008]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.896012] cron          D ffff8802fca96f30     0 170717  24244  309 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.896017]  ffff8803092f1d18 0000000000000082 ffff8803092f1d68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.896023]  ffff8802fca96f30 0000000000000000 0000000000000028 ffff8806e31b0638
Sep 16 13:10:26 node5 kernel: [29355.896030]  ffff8811a4624a38 ffff8802fca96f30 ffff8802fca974f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.896037] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.896041]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.896047]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.896052]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.896057]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.896062]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.896067]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.896071]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.896076]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.896081]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.896085]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.896090]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.896094] cron          D ffff882de260c2c0     0 170718  29653  932 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.896099]  ffff883da2f65788 0000000000000082 ffff883da2f65750 0000000000000282
Sep 16 13:10:26 node5 kernel: [29355.896106]  ffff883da2f65708 00000000020cf9a9 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.896113]  ffff8820737b0800 0000000101ba40e1 ffff882de260c888 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.896120] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.896129]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.896135]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.896158]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896183]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896207]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896231]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896257]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896283]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896318]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896322]  [<ffffffff81160a49>] ? zone_statistics+0x99/0xc0
Sep 16 13:10:26 node5 kernel: [29355.896357]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896391]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896425]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896458]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896491]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896524]  [<ffffffffa0350e61>] ? zfs_zaccess+0x1/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896558]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896592]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.896597]  [<ffffffff811bab82>] __lookup_hash+0x102/0x160
Sep 16 13:10:26 node5 kernel: [29355.896602]  [<ffffffff811bb50a>] lookup_hash+0x3a/0x50
Sep 16 13:10:26 node5 kernel: [29355.896607]  [<ffffffff811c0182>] do_filp_open+0x2e2/0xc50
Sep 16 13:10:26 node5 kernel: [29355.896612]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.896617]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.896621]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.896626]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.896631]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.896636]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.896641]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.896645] cron          D ffff883a21142340     0 170719  42294  526 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.896650]  ffff8836f6be3d18 0000000000000086 ffff8836f6be3d68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.896657]  ffff883a21142340 0000000000000000 0000000000000028 ffff8837928b8c38
Sep 16 13:10:26 node5 kernel: [29355.896664]  ffff8837928b9f38 ffff883a21142340 ffff883a21142908 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.896670] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.896674]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.896680]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.896686]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.896690]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.896695]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.896700]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.896705]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.896710]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.896714]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.896719]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.896724]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.896728] cron          D ffff8836eb2604c0     0 170720  42294  526 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.896733]  ffff883772fbfd18 0000000000000082 ffff883772fbfce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.896739]  ffff8836eb2604c0 000000000103de39 ffff882100c10770 ffff882000000001
Sep 16 13:10:26 node5 kernel: [29355.896746]  ffff882100c10760 0000000101ba40f0 ffff8836eb260a88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.896753] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.896757]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.896762]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.896768]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.896772]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.896777]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.896782]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.896787]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.896792]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.896796]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.896801]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.896806]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.896810] cron          D ffff8838ac58b3b0     0 170721  42294  526 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.896815]  ffff8837aad25d18 0000000000000082 ffff8837aad25ce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.896821]  ffff8838ac58b3b0 0000000000c8b311 ffff882100c10770 ffff882000000001
Sep 16 13:10:26 node5 kernel: [29355.896828]  ffff882100c10760 0000000101ba40f0 ffff8838ac58b978 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.896835] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.896839]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.896845]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.896850]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.896855]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.896860]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.896865]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.896869]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.896874]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.896879]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.896883]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.896888]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.896892] cron          D ffff8832d3f8d070     0 170722  42294  526 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.896897]  ffff882c37f13d18 0000000000000082 ffff882c37f13ce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.896904]  ffff883ef81fcd40 0000000000852585 ffff882100c10770 ffff882000000001
Sep 16 13:10:26 node5 kernel: [29355.896911]  ffff882100c10760 0000000101ba40f0 ffff8832d3f8d638 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.896917] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.896921]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.896927]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.896932]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.896936]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.896941]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.896946]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.896951]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.896956]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.896961]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.896965]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.896970]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.896974] cron          D ffff8832fb324b70     0 170723  42294  526 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.896979]  ffff8834f6117d18 0000000000000086 ffff8834f6117d68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.896986]  ffff8832fb324b70 0000000000000000 0000000000000028 ffff883e1cbf4148
Sep 16 13:10:26 node5 kernel: [29355.896993]  ffff883950cd3f38 ffff8832fb324b70 ffff8832fb325138 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.896999] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.897003]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.897009]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.897014]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.897019]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.897024]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.897029]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.897034]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.897039]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.897043]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.897048]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.897053]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.897057] cron          D ffff883cfc7ad230     0 170724  42294  526 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.897062]  ffff8837bbc97d18 0000000000000082 0000000000000000 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.897068]  ffff882dc0010c80 0000000000000000 0000000000000028 ffff883a74891638
Sep 16 13:10:26 node5 kernel: [29355.897075]  0000000000000000 0000000101ba40f0 ffff883cfc7ad7f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.897081] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.897085]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.897091]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.897097]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.897101]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.897106]  [<ffffffff810a7148>] ? up_read+0x18/0x30
Sep 16 13:10:26 node5 kernel: [29355.897111]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.897116]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.897120]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.897125]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.897130]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.897134]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.897139]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.897143] cron          D ffff882f30c76e70     0 170725  42294  526 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.897148]  ffff883ce13d7d18 0000000000000086 ffff883ce13d7d68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.897155]  ffff882f30c76e70 0000000000000000 0000000000000028 ffff882da2e6faa8
Sep 16 13:10:26 node5 kernel: [29355.897161]  ffff88394f802f38 ffff882f30c76e70 ffff882f30c77438 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.897168] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.897172]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.897178]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.897183]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.897188]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.897193]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.897198]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.897202]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.897207]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.897212]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.897216]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.897222]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.897225] cron          D ffff8802fca96440     0 170726  68064 1176 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.897230]  ffff8814b2ac5d18 0000000000000086 ffff8814b2ac5d68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.897237]  ffff8814b2ac5cc8 0000000000000000 0000000000000028 ffff881a624b00c0
Sep 16 13:10:26 node5 kernel: [29355.897244]  ffff881810776bb0 ffff8802fca96440 ffff8802fca96a08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.897250] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.897254]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.897259]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.897265]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.897270]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.897275]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.897280]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.897285]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.897290]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.897294]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.897299]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.897304]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.897308] cron          D ffff8837bd496eb0     0 170727 138064 1917 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.897313]  ffff88395114fd18 0000000000000082 ffff88395114fd68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.897320]  ffff8837bd496eb0 0000000000000000 0000000000000028 ffff88315a800e08
Sep 16 13:10:26 node5 kernel: [29355.897326]  ffff883613083e40 ffff8837bd496eb0 ffff8837bd497478 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.897333] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.897337]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.897343]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.897348]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.897352]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.897357]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.897362]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.897367]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.897372]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.897376]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.897381]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.897386]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.897390] cron          D ffff8812931d2140     0 170728  54449 1082 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.897395]  ffff8812285dfd18 0000000000000082 ffff8812285dfce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.897402]  ffff8812931d2140 0000000002afe07b ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.897409]  ffff8820737b0800 0000000101ba410e ffff8812931d2708 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.897415] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.897419]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.897425]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.897430]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.897435]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.897440]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.897445]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.897449]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.897454]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.897459]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.897463]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.897468]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.897472] cron          D ffff8838ac58a8c0     0 170729  12419  126 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.897478]  ffff883a895c3d18 0000000000000082 ffff883a895c3d68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.897484]  ffff8838ac58a8c0 0000000000000000 ffffffff8100beae ffff883a895c3d18
Sep 16 13:10:26 node5 kernel: [29355.897491]  0000000000000246 0000000000000000 ffff8838ac58ae88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.897497] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.897501]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.897506]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.897511]  [<ffffffff810559ed>] ? mutex_spin_on_owner+0x8d/0xc0
Sep 16 13:10:26 node5 kernel: [29355.897517]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.897522]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.897526]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.897531]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.897536]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.897541]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.897546]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.897551]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.897555]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.897560]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.897564] cron          D ffff8837964eeef0     0 170730  12419  126 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.897569]  ffff8839500837f8 0000000000000086 ffff8839500837c0 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.897576]  0000000000000000 000000000943a764 ffff88012bf50770 ffff882000000001
Sep 16 13:10:26 node5 kernel: [29355.897582]  ffff88012bf50760 0000000101ba4120 ffff8837964ef4b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.897589] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.897598]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.897605]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.897628]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.897653]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.897678]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.897702]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.897730]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.897760]  [<ffffffffa030aa88>] ? dsl_dir_tempreserve_impl+0x248/0x2b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.897787]  [<ffffffffa02ebf70>] dmu_object_alloc+0xf0/0x1c0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.897823]  [<ffffffffa0376f82>] zfs_mknode+0x112/0xde0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.897859]  [<ffffffffa0336841>] ? txg_rele_to_quiesce+0x11/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.897887]  [<ffffffffa02f74da>] ? dmu_tx_assign+0x37a/0x490 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.897923]  [<ffffffffa0370b6a>] zfs_create+0x5aa/0x700 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.897959]  [<ffffffffa0388575>] zpl_create+0xa5/0x1a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.897964]  [<ffffffff811ba983>] ? generic_permission+0x23/0xb0
Sep 16 13:10:26 node5 kernel: [29355.897969]  [<ffffffff811bc8c0>] vfs_create+0xd0/0xf0
Sep 16 13:10:26 node5 kernel: [29355.897973]  [<ffffffff811c091a>] do_filp_open+0xa7a/0xc50
Sep 16 13:10:26 node5 kernel: [29355.897979]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.897984]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.897989]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.897994]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.897999]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.898004]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.898009]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.898013] cron          D ffff882f30c76380     0 170731  26089  283 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.898018]  ffff883ca3245d18 0000000000000086 ffff883ca3245d68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.898025]  ffff882f30c76380 0000000000000000 0000000000000028 ffff883464db23a0
Sep 16 13:10:26 node5 kernel: [29355.898032]  ffff8833fd810c78 ffff882f30c76380 ffff882f30c76948 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.898038] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.898042]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.898049]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.898054]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.898059]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.898064]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.898069]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.898074]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.898079]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.898084]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.898088]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.898093]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.898097] cron          D ffff8833f4490f30     0 170732  26089  283 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.898102]  ffff8834106a1d18 0000000000000082 ffff8834106a1ce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.898109]  ffff8833f4490f30 000000001513099f ffff88012bf50770 ffff882000000001
Sep 16 13:10:26 node5 kernel: [29355.898116]  ffff88012bf50760 0000000101ba414d ffff8833f44914f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.898123] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.898126]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.898132]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.898138]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.898142]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.898147]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.898152]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.898157]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.898162]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.898166]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.898171]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.898176]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.898180] cron          D ffff8833f4490440     0 170733  26089  283 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.898185]  ffff8833f44d5d18 0000000000000086 ffff8833f44d5ce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.898191]  ffff8833f4490440 0000000000537628 ffff882100c10770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.898199]  ffff882100c10760 0000000101ba4122 ffff8833f4490a08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.898205] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.898209]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.898215]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.898220]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.898225]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.898230]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.898235]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.898239]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.898244]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.898249]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.898253]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.898258]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.898262] cron          D ffff882dc4ff4f70     0 170734  26089  283 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.898267]  ffff882dc4e45d18 0000000000000082 ffff882dc4e45ce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.898274]  ffff882dc4ff4f70 0000000018df1077 ffff88012bf50770 ffff882000000001
Sep 16 13:10:26 node5 kernel: [29355.898281]  ffff88012bf50760 0000000101ba4141 ffff882dc4ff5538 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.898287] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.898291]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.898297]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.898302]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.898307]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.898312]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.898317]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.898322]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.898327]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.898331]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.898336]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.898341]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.898344] cron          D ffff882dc4ff4480     0 170735  26089  283 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.898349]  ffff8837a4609d18 0000000000000086 ffff8837a4609ce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.898356]  ffff882dc4ff4480 0000000000537628 ffff882100c10770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.898363]  ffff882100c10760 0000000101ba4120 ffff882dc4ff4a48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.898369] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.898373]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.898379]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.898385]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.898389]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.898394]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.898399]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.898404]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.898408]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.898413]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.898418]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.898423]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.898427] cron          D ffff880592b94040     0 170736  33484  359 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.898431]  ffff880a04945a98 0000000000000082 0000000000000000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.898438]  0000000000000000 0000000000000002 ffff88012bed0770 000200da00000000
Sep 16 13:10:26 node5 kernel: [29355.898445]  ffff88012bed0760 0000000101ba4120 ffff880592b94608 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.898451] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.898460]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.898468]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.898491]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.898516]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.898541]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.898565]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.898593]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.898622]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.898647]  [<ffffffffa02ea8b9>] dmu_buf_hold_array+0x39/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.898652]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.898678]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.898714]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.898719]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.898754]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.898760]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.898764]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.898770]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.898774] cron          D ffff883228d10c70     0 170737 100205 1233 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.898779]  ffff8837a80a5d18 0000000000000086 ffff8837a80a5ce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.898786]  ffff883228d10c70 0000000000b83de8 ffff882100e90230 ffff882000000001
Sep 16 13:10:26 node5 kernel: [29355.898793]  ffff882100e90220 0000000101ba414d ffff883228d11238 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.898799] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.898803]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.898810]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.898815]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.898820]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.898825]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.898830]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.898835]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.898840]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.898844]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.898849]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.898855]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.898858] cron          D ffff883e30826fb0     0 170738  86735 1200 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.898864]  ffff8836c4ed1788 0000000000000082 0000000000000000 0000000000000282
Sep 16 13:10:26 node5 kernel: [29355.898871]  ffff8836c4ed1708 ffffffffa02f9159 ffff880d4a889640 ffff88206603cc60
Sep 16 13:10:26 node5 kernel: [29355.898878]  0000000000000000 0000000101ba414d ffff883e30827578 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.898884] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.898912]  [<ffffffffa02f9159>] ? spl_rw_clear_owner+0x39/0x50 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.898922]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.898929]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.898952]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.898976]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899000]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899025]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899050]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899076]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899111]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899116]  [<ffffffff81160a49>] ? zone_statistics+0x99/0xc0
Sep 16 13:10:26 node5 kernel: [29355.899151]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899185]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899219]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899253]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899286]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899291]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.899326]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899360]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899365]  [<ffffffff811bab82>] __lookup_hash+0x102/0x160
Sep 16 13:10:26 node5 kernel: [29355.899370]  [<ffffffff811bb50a>] lookup_hash+0x3a/0x50
Sep 16 13:10:26 node5 kernel: [29355.899374]  [<ffffffff811c0182>] do_filp_open+0x2e2/0xc50
Sep 16 13:10:26 node5 kernel: [29355.899379]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.899385]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.899389]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.899394]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.899399]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.899404]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.899409]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.899413] cron          D ffff8836c7848f70     0 170739 188076 3371 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.899418]  ffff883f4e7d7d18 0000000000000082 ffff883f4e7d7d68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.899425]  ffff8836c7848f70 0000000000000000 0000000000000028 ffff883d8f3857c8
Sep 16 13:10:26 node5 kernel: [29355.899432]  ffff8836ccddb6b8 ffff8836c7848f70 ffff8836c7849538 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.899438] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.899442]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.899449]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.899454]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.899459]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.899464]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.899469]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.899474]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.899479]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.899484]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.899488]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.899493]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.899497] cron          D ffff882e3aaa41c0     0 170740 188076 3371 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.899502]  ffff883f30dd1d18 0000000000000082 ffff883f30dd1d68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.899509]  ffff882e3aaa41c0 0000000000000000 0000000000000028 ffff883ce6ca40c0
Sep 16 13:10:26 node5 kernel: [29355.899515]  ffff883f4432e6b8 ffff882e3aaa41c0 ffff882e3aaa4788 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.899522] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.899526]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.899532]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.899537]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.899542]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.899547]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.899552]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.899557]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.899561]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.899566]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.899571]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.899576]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.899580] crond         D ffff8837bd4963c0     0 170741 197673 2223 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.899585]  ffff883b459afab8 0000000000000086 ffff883b459afa48 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.899591]  0000000000000282 ffff88047b7d2930 ffff883b459afa48 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.899598]  0000000000004030 ffff884071850000 ffff8837bd496988 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.899605] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.899639]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899648]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.899655]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.899678]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899703]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899728]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899752]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899778]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899807]  [<ffffffffa02ea8e5>] dmu_buf_hold_array+0x65/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899815]  [<ffffffffa002a131>] ? avl_find+0x1/0xb0 [zavl]
Sep 16 13:10:26 node5 kernel: [29355.899840]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899876]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899880]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.899914]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.899920]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.899925]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.899930]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.899934] crond         D ffff88202035cf70     0 170742 143067 1536 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.899939]  ffff882020373ab8 0000000000000082 ffff882020373a80 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.899946]  ffff883b87e926f8 0000000000902156 ffff88012be50770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.899953]  ffff88012be50760 0000000101ba4172 ffff88202035d538 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.899960] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.899969]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.899975]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.899999]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900024]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900048]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900072]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900098]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900127]  [<ffffffffa02ea8e5>] dmu_buf_hold_array+0x65/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900153]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900188]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900193]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.900227]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900232]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.900237]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.900243]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.900247] cron          D ffff8830935feb70     0 170743  32006  211 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.900252]  ffff8833d2a99688 0000000000000086 ffff883fdb828d60 0000000000000082
Sep 16 13:10:26 node5 kernel: [29355.900258]  ffff8833d2a99688 ffffffff810a6ad3 ffff882100ddece8 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.900265]  0000000000000001 ffff882100ddece8 ffff8830935ff138 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.900272] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.900276]  [<ffffffff810a6ad3>] ? __hrtimer_start_range_ns+0x1a3/0x460
Sep 16 13:10:26 node5 kernel: [29355.900282]  [<ffffffff81062273>] ? perf_event_task_sched_out+0x33/0x70
Sep 16 13:10:26 node5 kernel: [29355.900292]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.900326]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900332]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.900356]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900381]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900385]  [<ffffffff81160a49>] ? zone_statistics+0x99/0xc0
Sep 16 13:10:26 node5 kernel: [29355.900410]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900433]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900462]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900490]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900516]  [<ffffffffa02eb213>] dmu_read+0x33/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900521]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.900556]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900590]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900595]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.900600]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.900633]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900666]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900671]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.900676]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.900680]  [<ffffffff8114d635>] ondemand_readahead+0x115/0x240
Sep 16 13:10:26 node5 kernel: [29355.900685]  [<ffffffff8114d8d6>] page_cache_sync_readahead+0x86/0xa0
Sep 16 13:10:26 node5 kernel: [29355.900691]  [<ffffffff811375fb>] filemap_fault+0x5bb/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.900696]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.900709]  [<ffffffffa020b37f>] ? tsd_exit+0x5f/0x2b0 [spl]
Sep 16 13:10:26 node5 kernel: [29355.900714]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.900719]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.900724]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.900729]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.900734]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.900739]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.900744]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.900749]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.900753] cron          D ffff88202035c480     0 170744  46701 1055 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.900758]  ffff881530f87698 0000000000000082 ffff881530f87660 ffff88331bdb5000
Sep 16 13:10:26 node5 kernel: [29355.900764]  ffff8833146955e0 000000000250ba15 ffff882100cd0770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.900771]  ffff882100cd0760 0000000101ba41a9 ffff88202035ca48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.900778] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.900786]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.900794]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.900816]  [<ffffffffa02d57c7>] buf_hash_find+0x97/0x130 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900839]  [<ffffffffa02dc547>] arc_read+0xb7/0x910 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900862]  [<ffffffffa02dfb80>] ? dbuf_create+0x2a0/0x3e0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900886]  [<ffffffffa02e06e0>] ? dbuf_read_done+0x0/0x110 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900910]  [<ffffffffa02e1360>] dbuf_read+0x1f0/0x770 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900936]  [<ffffffffa02ea269>] dmu_buf_hold_array_by_dnode+0x139/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900965]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.900970]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.901005]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901039]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901045]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.901050]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.901083]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901115]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901120]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.901125]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.901130]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.901135]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.901140]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.901145]  [<ffffffff811d0cd0>] ? mntput_no_expire+0x30/0x110
Sep 16 13:10:26 node5 kernel: [29355.901150]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.901155]  [<ffffffff811bbab9>] ? putname+0x29/0x40
Sep 16 13:10:26 node5 kernel: [29355.901159]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.901164]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.901169]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.901174]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.901178] cron          D ffff88054b88c940     0 170745  81536 1088 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.901183]  ffff88151faabd18 0000000000000082 ffff88151faabd68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.901190]  ffff88054b88c940 0000000000000000 0000000000000020 ffff880c378bb5f8
Sep 16 13:10:26 node5 kernel: [29355.901197]  ffff883e31416188 ffff88054b88c940 ffff88054b88cf08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.901203] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.901207]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.901214]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.901219]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.901224]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.901229]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.901235]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.901239]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.901244]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.901249]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.901254]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.901259]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.901262] cron          D ffff88054b88d430     0 170746  81536 1088 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.901268]  ffff88150ed3bd18 0000000000000086 ffff88150ed3bd68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.901275]  ffff88054b88d430 0000000000000000 0000000000000028 ffff880f44866bb0
Sep 16 13:10:26 node5 kernel: [29355.901281]  ffff8832db1c5188 ffff88054b88d430 ffff88054b88d9f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.901288] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.901292]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.901298]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.901304]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.901308]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.901315]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.901320]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.901324]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.901329]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.901334]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.901338]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.901343]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.901347] cron          D ffff883daaef0ff0     0 170747  79246 1187 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.901352]  ffff88322c7f1d18 0000000000000082 ffff88322c7f1ce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.901359]  ffff883daaef0ff0 000000000060e773 ffff88012bf50770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.901366]  ffff88012bf50760 0000000101ba41ea ffff883daaef15b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.901372] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.901376]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.901382]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.901388]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.901392]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.901397]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.901402]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.901407]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.901412]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.901416]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.901421]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.901426]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.901430] cron          D ffff88378dce3030     0 170748 125202 1535 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.901435]  ffff883ef5349d18 0000000000000086 ffff883ef5349d68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.901441]  ffff88378dce3030 0000000000000000 0000000000000028 ffff8833d7436188
Sep 16 13:10:26 node5 kernel: [29355.901448]  ffff88363dc1aa58 ffff88378dce3030 ffff88378dce35f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.901455] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.901459]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.901465]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.901470]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.901475]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.901480]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.901485]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.901489]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.901494]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.901499]  [<ffffffff811cda91>] ? alloc_fd+0x1/0x160
Sep 16 13:10:26 node5 kernel: [29355.901503]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.901508]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.901513]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.901517] cron          D ffff88117cbbe840     0 170749 114928 1303 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.901522]  ffff881927a89698 0000000000000082 ffff881927a89638 0000000100000000
Sep 16 13:10:26 node5 kernel: [29355.901529]  ffff884070db0800 0004523000000001 ffff884070db07e0 ffff884070db07d0
Sep 16 13:10:26 node5 kernel: [29355.901536]  ffff884072652780 ffff88404ce20000 ffff88117cbbee08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.901542] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.901551]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.901559]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.901581]  [<ffffffffa02d57c7>] buf_hash_find+0x97/0x130 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901604]  [<ffffffffa02dc547>] arc_read+0xb7/0x910 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901627]  [<ffffffffa02dfb64>] ? dbuf_create+0x284/0x3e0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901651]  [<ffffffffa02dee4a>] ? dbuf_find+0x6a/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901674]  [<ffffffffa02e06e0>] ? dbuf_read_done+0x0/0x110 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901698]  [<ffffffffa02e1360>] dbuf_read+0x1f0/0x770 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901724]  [<ffffffffa02ea269>] dmu_buf_hold_array_by_dnode+0x139/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901752]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901757]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.901792]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901826]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901832]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.901837]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.901870]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901903]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.901907]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.901912]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.901917]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.901922]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.901927]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.901932]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.901937]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.901942]  [<ffffffff81291c26>] ? prio_tree_insert+0x256/0x2b0
Sep 16 13:10:26 node5 kernel: [29355.901947]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.901952]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.901957]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.901962]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.901967]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.901971] cron          D ffff88157c2dcfb0     0 170750  66747 1177 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.901976]  ffff8815205d9ab8 0000000000000086 ffff8815205d9a48 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.901983]  0000000000000286 ffff8836924f9250 ffff8815205d9a48 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.901990]  0000000000004030 ffff884071850000 ffff88157c2dd578 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.901996] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.902029]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902034]  [<ffffffff810559fb>] ? mutex_spin_on_owner+0x9b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.902043]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.902050]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.902073]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902098]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902123]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902147]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902173]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902202]  [<ffffffffa02ea8e5>] dmu_buf_hold_array+0x65/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902228]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902263]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902268]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.902301]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902307]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.902312]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.902317]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.902321] cron          D ffff88157c2dc4c0     0 170751  66747 1177 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.902326]  ffff881531111ab8 0000000000000082 ffff881531111a48 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.902333]  0000000000000282 ffff8812a4a41150 ffff881531111a48 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.902340]  0000000000004030 ffff884071850000 ffff88157c2dca88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.902347] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.902379]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902388]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.902395]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.902418]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902443]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902467]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902491]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902517]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902546]  [<ffffffffa02ea8e5>] dmu_buf_hold_array+0x65/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902572]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902607]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902611]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.902645]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902650]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.902655]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.902661]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.902664] cron          D ffff88379781c100     0 170752 146079 2028 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.902670]  ffff883dc015fd18 0000000000000086 ffff883dc015fd68 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.902676]  ffffffff815324d2 0000000000000000 0000000000000028 ffff883bed0b6788
Sep 16 13:10:26 node5 kernel: [29355.902683]  ffff883e314177c8 ffff88379781c100 ffff88379781c6c8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.902690] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.902694]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.902698]  [<ffffffff815324d2>] ? _spin_lock+0x12/0x30
Sep 16 13:10:26 node5 kernel: [29355.902704]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.902710]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.902715]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.902720]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.902725]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.902730]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.902735]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.902740]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.902745]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.902750]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.902753] cron          D ffff88107db00f30     0 170753  35255  185 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.902759]  ffff881531107788 0000000000000086 0000000000000000 0000000000000282
Sep 16 13:10:26 node5 kernel: [29355.902765]  ffff881531107708 ffffffffa02f9159 ffff8820737b0810 ffff88206f83e2d0
Sep 16 13:10:26 node5 kernel: [29355.902772]  ffff8820737b0800 0000000101ba423d ffff88107db014f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.902778] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.902806]  [<ffffffffa02f9159>] ? spl_rw_clear_owner+0x39/0x50 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902817]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.902823]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.902846]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902871]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902895]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902919]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902945]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.902971]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903005]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903010]  [<ffffffff81160a49>] ? zone_statistics+0x99/0xc0
Sep 16 13:10:26 node5 kernel: [29355.903045]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903078]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903112]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903145]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903178]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903211]  [<ffffffffa0350e61>] ? zfs_zaccess+0x1/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903245]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903279]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903284]  [<ffffffff811bab82>] __lookup_hash+0x102/0x160
Sep 16 13:10:26 node5 kernel: [29355.903289]  [<ffffffff811bb50a>] lookup_hash+0x3a/0x50
Sep 16 13:10:26 node5 kernel: [29355.903293]  [<ffffffff811c0182>] do_filp_open+0x2e2/0xc50
Sep 16 13:10:26 node5 kernel: [29355.903299]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.903304]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.903309]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.903313]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.903318]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.903323]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.903328]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.903332] cron          D ffff8813cf1b4eb0     0 170754 178728 2956 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.903337]  ffff8802133a57a8 0000000000000082 ffff88052e9394c0 ffff8820659be930
Sep 16 13:10:26 node5 kernel: [29355.903344]  ffff8802133a57d8 ffffffffa02f9ae6 0000000000010206 ffffffff00000001
Sep 16 13:10:26 node5 kernel: [29355.903351]  ffff8820659be938 ffff8820659be930 ffff8813cf1b5478 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.903357] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.903385]  [<ffffffffa02f9ae6>] ? dmu_zfetch+0x656/0xe30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903394]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.903403]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.903426]  [<ffffffffa02d75fa>] arc_buf_add_ref+0x9a/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903450]  [<ffffffffa02e1946>] __dbuf_hold_impl+0x66/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903473]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903497]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903523]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903549]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903583]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903616]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903649]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903681]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903714]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903746]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903778]  [<ffffffffa0350e61>] ? zfs_zaccess+0x1/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903812]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903846]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903850]  [<ffffffff811bab82>] __lookup_hash+0x102/0x160
Sep 16 13:10:26 node5 kernel: [29355.903855]  [<ffffffff811bb50a>] lookup_hash+0x3a/0x50
Sep 16 13:10:26 node5 kernel: [29355.903860]  [<ffffffff811c0182>] do_filp_open+0x2e2/0xc50
Sep 16 13:10:26 node5 kernel: [29355.903865]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.903870]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.903875]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.903880]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.903884]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.903889]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.903894]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.903898] cron          D ffff883092951070     0 170755  36697 2842 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.903903]  ffff883f46c3d6f8 0000000000000086 ffff883f46c3d6c0 0000000000000007
Sep 16 13:10:26 node5 kernel: [29355.903910]  ffff88047b7d2080 00000000009d8a15 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.903917]  ffff8820737b0800 0000000101ba429e ffff883092951638 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.903924] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.903933]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.903938]  [<ffffffff815317d6>] ? down_read+0x16/0x30
Sep 16 13:10:26 node5 kernel: [29355.903944]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.903967]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.903992]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904016]  [<ffffffffa02e1a4d>] __dbuf_hold_impl+0x16d/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904041]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904065]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904091]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904119]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904124]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.904159]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904193]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904198]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.904203]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.904236]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904269]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904274]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.904279]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.904284]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.904290]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.904295]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.904300]  [<ffffffff811d0cd0>] ? mntput_no_expire+0x30/0x110
Sep 16 13:10:26 node5 kernel: [29355.904313]  [<ffffffffa020ae89>] ? tsd_hash_search+0x99/0x200 [spl]
Sep 16 13:10:26 node5 kernel: [29355.904318]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.904323]  [<ffffffff811bbab9>] ? putname+0x29/0x40
Sep 16 13:10:26 node5 kernel: [29355.904327]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.904332]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.904337]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.904342]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.904346] cron          D ffff883e1cca85c0     0 170756 189218 3297 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.904351]  ffff883ec1817b28 0000000000000082 ffff883ec1817ac8 0000000000000030
Sep 16 13:10:26 node5 kernel: [29355.904358]  0000000000000018 ffffffffa0320105 ffff883ec1817ab8 ffffffff815317d6
Sep 16 13:10:26 node5 kernel: [29355.904365]  ffff883ec1817ac8 ffffffff815317d6 ffff883e1cca8b88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.904371] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.904403]  [<ffffffffa0320105>] ? sa_attr_op+0x245/0x3e0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904408]  [<ffffffff815317d6>] ? down_read+0x16/0x30
Sep 16 13:10:26 node5 kernel: [29355.904412]  [<ffffffff815317d6>] ? down_read+0x16/0x30
Sep 16 13:10:26 node5 kernel: [29355.904421]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.904427]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.904450]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904475]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904500]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904524]  [<ffffffffa02e237f>] dbuf_hold_level+0x1f/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904552]  [<ffffffffa02f782a>] dmu_tx_check_ioerr+0x4a/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904579]  [<ffffffffa02f81e7>] dmu_tx_hold_zap+0x167/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904613]  [<ffffffffa0370223>] zfs_remove+0xa3/0x440 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904618]  [<ffffffff811d0cd0>] ? mntput_no_expire+0x30/0x110
Sep 16 13:10:26 node5 kernel: [29355.904652]  [<ffffffffa0387ea6>] zpl_unlink+0x46/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904657]  [<ffffffff811bc02d>] vfs_unlink+0x8d/0xd0
Sep 16 13:10:26 node5 kernel: [29355.904661]  [<ffffffff811bb50a>] ? lookup_hash+0x3a/0x50
Sep 16 13:10:26 node5 kernel: [29355.904666]  [<ffffffff811bf343>] do_unlinkat+0x163/0x260
Sep 16 13:10:26 node5 kernel: [29355.904671]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.904676]  [<ffffffff811bf456>] sys_unlink+0x16/0x20
Sep 16 13:10:26 node5 kernel: [29355.904681]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.904685] cron          D ffff882f0ba8b0b0     0 170757  13987  221 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.904691]  ffff883a2efa5d18 0000000000000086 0000000000000000 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.904697]  ffff882f0ba8b0b0 0000000000000000 0000000000000028 ffff88382d1aa7c8
Sep 16 13:10:26 node5 kernel: [29355.904704]  0000000000000000 0000000101ba42cf ffff882f0ba8b678 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.904711] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.904715]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.904721]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.904727]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.904731]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.904736]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.904742]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.904746]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.904751]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.904756]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.904761]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.904766]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.904770] cron          D ffff882f0ba8a5c0     0 170758  72192 1170 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.904775]  ffff8832cdf77d18 0000000000000082 0000000000000000 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.904782]  ffff882f0ba8a5c0 0000000000000000 0000000000000028 ffff882d77e54850
Sep 16 13:10:26 node5 kernel: [29355.904788]  0000000000000000 0000000101ba428d ffff882f0ba8ab88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.904795] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.904799]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.904805]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.904811]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.904815]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.904821]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.904826]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.904830]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.904835]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.904840]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.904844]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.904849]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.904853] cron          D ffff883edec4a3c0     0 170759  33181  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.904858]  ffff883dcfe45ab8 0000000000000082 0000000000000000 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.904865]  0000000000000282 ffff8812a4a404c0 ffff883dcfe45a48 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.904872]  0000000000000000 0000000101ba42e0 ffff883edec4a988 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.904878] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.904911]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904921]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.904927]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.904950]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.904975]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905000]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905025]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905050]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905080]  [<ffffffffa02ea8e5>] dmu_buf_hold_array+0x65/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905088]  [<ffffffffa002a9c8>] ? avl_add+0x38/0x50 [zavl]
Sep 16 13:10:26 node5 kernel: [29355.905113]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905119]  [<ffffffff811360ca>] ? find_lock_page+0x2a/0x80
Sep 16 13:10:26 node5 kernel: [29355.905153]  [<ffffffffa03717c6>] zfs_read+0x2e6/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905186]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905192]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.905197]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.905202]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.905206] cron          D ffff8837964ee400     0 170760 1002286 2029 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.905211]  ffff882d65e7bd18 0000000000000082 ffff882d65e7bce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.905218]  ffff8837964ee400 0000000000bbf343 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.905225]  ffff8820737b0800 0000000101ba42dd ffff8837964ee9c8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.905232] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.905236]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.905242]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.905248]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.905252]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.905257]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.905262]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.905267]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.905272]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.905277]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.905282]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.905287]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.905290] cron          D ffff883ef31630f0     0 170761 1002286 2029 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.905296]  ffff882da56a9d18 0000000000000086 ffff882da56a9ce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.905303]  ffff883ef31630f0 0000000000bc9918 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.905310]  ffff8820737b0800 0000000101ba4241 ffff883ef31636b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.905316] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.905320]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.905326]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.905332]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.905336]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.905341]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.905346]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.905351]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.905356]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.905360]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.905365]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.905370]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.905374] cron          D ffff883e308264c0     0 170762  14009  193 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.905379]  ffff8836bc177688 0000000000000086 0000000000000000 00000000fffffffe
Sep 16 13:10:26 node5 kernel: [29355.905386]  010017003004012a 7502ffff00000000 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.905393]  ffffffffffffffff ffffffffffffffff ffff883e30826a88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.905399] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.905408]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.905415]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.905438]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905463]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905468]  [<ffffffff81160a49>] ? zone_statistics+0x99/0xc0
Sep 16 13:10:26 node5 kernel: [29355.905492]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905516]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905545]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905573]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905598]  [<ffffffffa02eb213>] dmu_read+0x33/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905604]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.905639]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905672]  [<ffffffffa0371991>] ? zfs_getpage+0x1/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905706]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905711]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.905716]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.905749]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905782]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905787]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.905792]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.905796]  [<ffffffff8114d635>] ondemand_readahead+0x115/0x240
Sep 16 13:10:26 node5 kernel: [29355.905801]  [<ffffffff8114d8d6>] page_cache_sync_readahead+0x86/0xa0
Sep 16 13:10:26 node5 kernel: [29355.905806]  [<ffffffff811375fb>] filemap_fault+0x5bb/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.905811]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.905816]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.905821]  [<ffffffff811d0cd0>] ? mntput_no_expire+0x30/0x110
Sep 16 13:10:26 node5 kernel: [29355.905834]  [<ffffffffa020ae89>] ? tsd_hash_search+0x99/0x200 [spl]
Sep 16 13:10:26 node5 kernel: [29355.905839]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.905844]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.905849]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.905855]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.905859]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.905863] cron          D ffff883759eb3130     0 170763  33649  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.905868]  ffff883765b7ba08 0000000000000086 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.905875]  ffff8820720b7900 00045230a538ec83 ffff88012be50770 ffff883765b7bb18
Sep 16 13:10:26 node5 kernel: [29355.905882]  ffff88012be50760 0000000101ba4307 ffff883759eb36f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.905889] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.905899]  [<ffffffffa01fe041>] ? spl_kmem_cache_alloc+0xa1/0x1090 [spl]
Sep 16 13:10:26 node5 kernel: [29355.905909]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.905917]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.905939]  [<ffffffffa02d57c7>] buf_hash_find+0x97/0x130 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905962]  [<ffffffffa02dc547>] arc_read+0xb7/0x910 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905985]  [<ffffffffa02dfb64>] ? dbuf_create+0x284/0x3e0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.905991]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.906014]  [<ffffffffa02e06e0>] ? dbuf_read_done+0x0/0x110 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906038]  [<ffffffffa02e1360>] dbuf_read+0x1f0/0x770 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906065]  [<ffffffffa02ea269>] dmu_buf_hold_array_by_dnode+0x139/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906094]  [<ffffffffa02ea8e5>] dmu_buf_hold_array+0x65/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906102]  [<ffffffffa002a9c8>] ? avl_add+0x38/0x50 [zavl]
Sep 16 13:10:26 node5 kernel: [29355.906127]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906162]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906167]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.906180]  [<ffffffffa020aae3>] ? crhold+0x13/0x20 [spl]
Sep 16 13:10:26 node5 kernel: [29355.906214]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906219]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.906224]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.906230]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.906234] cron          D ffff8805c49d4ff0     0 170764 103396 1088 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.906239]  ffff88021ffdf918 0000000000000086 0000000000000000 ffffffffa01fddab
Sep 16 13:10:26 node5 kernel: [29355.906246]  0000000000000014 ffff88015f832838 ffff8820737b0810 ffff881a21994660
Sep 16 13:10:26 node5 kernel: [29355.906253]  ffff8820737b0800 0000000101ba4308 ffff8805c49d55b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.906259] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.906270]  [<ffffffffa01fddab>] ? kmem_free_debug+0x4b/0x150 [spl]
Sep 16 13:10:26 node5 kernel: [29355.906281]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.906287]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.906310]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906336]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906360]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906385]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906413]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906425]  [<ffffffffa02001cf>] ? kmem_alloc_debug+0x8f/0x4c0 [spl]
Sep 16 13:10:26 node5 kernel: [29355.906453]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906480]  [<ffffffffa02f67c3>] dmu_tx_hold_object_impl+0x33/0xf0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906507]  [<ffffffffa02f80b9>] dmu_tx_hold_zap+0x39/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906534]  [<ffffffffa02f85e4>] dmu_tx_hold_sa_create+0x114/0x120 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906568]  [<ffffffffa03707cb>] zfs_create+0x20b/0x700 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906603]  [<ffffffffa0388575>] zpl_create+0xa5/0x1a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906607]  [<ffffffff811ba983>] ? generic_permission+0x23/0xb0
Sep 16 13:10:26 node5 kernel: [29355.906612]  [<ffffffff811bc8c0>] vfs_create+0xd0/0xf0
Sep 16 13:10:26 node5 kernel: [29355.906617]  [<ffffffff811c091a>] do_filp_open+0xa7a/0xc50
Sep 16 13:10:26 node5 kernel: [29355.906622]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.906627]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.906632]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.906637]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.906642]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.906646]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.906652]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.906656] cron          D ffff880f5a699030     0 170765 106602 1287 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.906662]  ffff8814018cf7f8 0000000000000082 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.906668]  0000000000000000 0000000000000000 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.906675]  0000000000000000 0000000000000000 ffff880f5a6995f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.906681] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.906709]  [<ffffffffa02f97bb>] ? dmu_zfetch+0x32b/0xe30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906718]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.906725]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.906748]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906754]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.906777]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906802]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906826]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906854]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906884]  [<ffffffffa030aa88>] ? dsl_dir_tempreserve_impl+0x248/0x2b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906910]  [<ffffffffa02ebf70>] dmu_object_alloc+0xf0/0x1c0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906944]  [<ffffffffa0376f82>] zfs_mknode+0x112/0xde0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.906979]  [<ffffffffa0336841>] ? txg_rele_to_quiesce+0x11/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907007]  [<ffffffffa02f74da>] ? dmu_tx_assign+0x37a/0x490 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907041]  [<ffffffffa0370b6a>] zfs_create+0x5aa/0x700 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907076]  [<ffffffffa0388575>] zpl_create+0xa5/0x1a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907081]  [<ffffffff811ba983>] ? generic_permission+0x23/0xb0
Sep 16 13:10:26 node5 kernel: [29355.907086]  [<ffffffff811bc8c0>] vfs_create+0xd0/0xf0
Sep 16 13:10:26 node5 kernel: [29355.907090]  [<ffffffff811c091a>] do_filp_open+0xa7a/0xc50
Sep 16 13:10:26 node5 kernel: [29355.907096]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.907101]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.907105]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.907110]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.907115]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.907120]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.907125]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.907129] cron          D ffff883092950580     0 170766 135250 1585 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.907134]  ffff882dd45c7d18 0000000000000082 ffff882dd45c7ce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.907141]  ffff883092950580 000000000108bfb8 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.907148]  ffff8820737b0800 0000000101ba4308 ffff883092950b48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.907154] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.907158]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.907165]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.907172]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.907176]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.907181]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.907186]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.907191]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.907196]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.907201]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.907205]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.907211]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.907214] cron          D ffff88322e3bd170     0 170767 135250 1585 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.907219]  ffff883d8d5ebd18 0000000000000086 ffff883d8d5ebce0 ffffffff811bd751
Sep 16 13:10:26 node5 kernel: [29355.907226]  ffff88322e3bd170 000000000107ee56 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.907233]  ffff8820737b0800 0000000101ba4367 ffff88322e3bd738 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.907240] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.907243]  [<ffffffff811bd751>] ? __link_path_walk+0x1a1/0x1090
Sep 16 13:10:26 node5 kernel: [29355.907249]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.907255]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.907259]  [<ffffffff811c017a>] do_filp_open+0x2da/0xc50
Sep 16 13:10:26 node5 kernel: [29355.907264]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.907269]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.907274]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.907279]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.907284]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.907288]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.907293]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.907297] sshd          D ffff8812931d2c30     0 170768  55345 1091 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.907302]  ffff8818372d3718 0000000000000086 ffff8818372d3668 ffff884071850000
Sep 16 13:10:26 node5 kernel: [29355.907309]  0000000000004030 ffff884070efa3c0 ffff8818372d36b8 ffffffffa0209f1d
Sep 16 13:10:26 node5 kernel: [29355.907315]  ffff8818372d36d8 0000000000000246 ffff8812931d31f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.907322] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.907334]  [<ffffffffa0209f1d>] ? __cv_init+0x4d/0x100 [spl]
Sep 16 13:10:26 node5 kernel: [29355.907346]  [<ffffffffa01fe03a>] ? spl_kmem_cache_alloc+0x9a/0x1090 [spl]
Sep 16 13:10:26 node5 kernel: [29355.907351]  [<ffffffff8100bc4e>] ? apic_timer_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.907360]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.907388]  [<ffffffffa02f9942>] ? dmu_zfetch+0x4b2/0xe30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907396]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.907419]  [<ffffffffa02d75fa>] arc_buf_add_ref+0x9a/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907443]  [<ffffffffa02e1946>] __dbuf_hold_impl+0x66/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907467]  [<ffffffffa02e1a4d>] __dbuf_hold_impl+0x16d/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907492]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907516]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907542]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907569]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907574]  [<ffffffff81292ac3>] ? radix_tree_preload+0x23/0xa0
Sep 16 13:10:26 node5 kernel: [29355.907579]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.907614]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907648]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907654]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.907659]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.907692]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907725]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907730]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.907735]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.907740]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.907745]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.907750]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.907755]  [<ffffffff811ccd5d>] ? notify_change+0x2bd/0x340
Sep 16 13:10:26 node5 kernel: [29355.907759]  [<ffffffff81175e81>] ? page_remove_rmap+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.907765]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.907769]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.907774]  [<ffffffff811c8168>] ? d_free+0x58/0x60
Sep 16 13:10:26 node5 kernel: [29355.907778]  [<ffffffff811c81c4>] ? d_kill+0x54/0x70
Sep 16 13:10:26 node5 kernel: [29355.907782]  [<ffffffff811d0cd0>] ? mntput_no_expire+0x30/0x110
Sep 16 13:10:26 node5 kernel: [29355.907788]  [<ffffffff811ae82c>] ? __fput+0x1ec/0x280
Sep 16 13:10:26 node5 kernel: [29355.907793]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.907797]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.907801] ps            D ffff881547535070     0 170769  72125 1088 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.907806]  ffff88064fa416f8 0000000000000086 ffff884073fce280 0004523000000000
Sep 16 13:10:26 node5 kernel: [29355.907813]  ffff884073fce260 ffff884073fce250 0000000000000015 ffff884071850000
Sep 16 13:10:26 node5 kernel: [29355.907820]  0000000000004030 ffff884070efa3c0 ffff881547535638 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.907826] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.907859]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907868]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.907873]  [<ffffffff815317d6>] ? down_read+0x16/0x30
Sep 16 13:10:26 node5 kernel: [29355.907879]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.907902]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907927]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907951]  [<ffffffffa02e1a4d>] __dbuf_hold_impl+0x16d/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.907976]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908000]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908026]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908053]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908058]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.908093]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908126]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908132]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.908137]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.908169]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908201]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908206]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.908211]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.908216]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.908221]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.908226]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.908232]  [<ffffffff8118c3ea>] ? alloc_pages_current+0xaa/0x110
Sep 16 13:10:26 node5 kernel: [29355.908237]  [<ffffffff8104eb17>] ? pte_alloc_one+0x37/0x50
Sep 16 13:10:26 node5 kernel: [29355.908241]  [<ffffffff811679a2>] ? __pte_alloc+0xd2/0x350
Sep 16 13:10:26 node5 kernel: [29355.908246]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.908251]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.908256]  [<ffffffff811b4699>] ? free_bprm+0x59/0x60
Sep 16 13:10:26 node5 kernel: [29355.908261]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.908266]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.908270] ps            D ffff881547534580     0 170771 110072 1281 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.908275]  ffff88124009f358 0000000000000082 ffff880da924bfa0 6e756f436c696108
Sep 16 13:10:26 node5 kernel: [29355.908282]  ffff88124009f2e8 ffffffffa0209f1d ffff88124009f308 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.908289]  9400000000000008 ffff882efac364c0 ffff881547534b48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.908295] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.908308]  [<ffffffffa0209f1d>] ? __cv_init+0x4d/0x100 [spl]
Sep 16 13:10:26 node5 kernel: [29355.908313]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.908322]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.908329]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.908352]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908376]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908381]  [<ffffffff81160a49>] ? zone_statistics+0x99/0xc0
Sep 16 13:10:26 node5 kernel: [29355.908405]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908429]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908457]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908462]  [<ffffffff81160a49>] ? zone_statistics+0x99/0xc0
Sep 16 13:10:26 node5 kernel: [29355.908490]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908515]  [<ffffffffa02eb213>] dmu_read+0x33/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908521]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.908555]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908590]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908595]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.908600]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.908633]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908666]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908671]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.908679]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.908684]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.908689]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.908694]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.908699]  [<ffffffff8118c3ea>] ? alloc_pages_current+0xaa/0x110
Sep 16 13:10:26 node5 kernel: [29355.908704]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.908710]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.908714]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.908720]  [<ffffffff8117003c>] ? __vma_link_file+0x4c/0x80
Sep 16 13:10:26 node5 kernel: [29355.908725]  [<ffffffff8117014b>] ? vma_link+0x9b/0xf0
Sep 16 13:10:26 node5 kernel: [29355.908730]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.908734]  [<ffffffff81165fbe>] ? free_pgtables+0xce/0x150
Sep 16 13:10:26 node5 kernel: [29355.908739]  [<ffffffff8114f2eb>] ? lru_add_drain_cpu+0x8b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.908744]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.908749]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.908755]  [<ffffffff8129adcf>] ? __clear_user+0x3f/0x70
Sep 16 13:10:26 node5 kernel: [29355.908759]  [<ffffffff8129adb1>] ? __clear_user+0x21/0x70
Sep 16 13:10:26 node5 kernel: [29355.908764]  [<ffffffff8129ae38>] clear_user+0x38/0x40
Sep 16 13:10:26 node5 kernel: [29355.908769]  [<ffffffff8120be9d>] padzero+0x2d/0x40
Sep 16 13:10:26 node5 kernel: [29355.908773]  [<ffffffff8120dc57>] load_elf_binary+0x1987/0x1a10
Sep 16 13:10:26 node5 kernel: [29355.908778]  [<ffffffff81165853>] ? follow_page+0x413/0x500
Sep 16 13:10:26 node5 kernel: [29355.908783]  [<ffffffff8116c6e0>] ? __get_user_pages+0x110/0x410
Sep 16 13:10:26 node5 kernel: [29355.908788]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.908794]  [<ffffffff811b60f7>] search_binary_handler+0x127/0x370
Sep 16 13:10:26 node5 kernel: [29355.908799]  [<ffffffff811b6aa3>] do_execve+0x223/0x2d0
Sep 16 13:10:26 node5 kernel: [29355.908805]  [<ffffffff8100966a>] sys_execve+0x4a/0x80
Sep 16 13:10:26 node5 kernel: [29355.908809]  [<ffffffff8100b55a>] stub_execve+0x6a/0xc0
Sep 16 13:10:26 node5 kernel: [29355.908813] httpd.itk     D ffff8817a560e280     0 170772 169877 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.908818]  ffff88141537dab8 0000000000000082 ffff88141537da48 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.908825]  ffff884071850000 ffff880938498050 ffff88141537da48 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.908832]  0000000000004030 ffff884071850000 ffff8817a560e848 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.908838] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.908871]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908881]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.908888]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.908911]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908936]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908961]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.908985]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909011]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909040]  [<ffffffffa02ea8e5>] dmu_buf_hold_array+0x65/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909048]  [<ffffffffa002a131>] ? avl_find+0x1/0xb0 [zavl]
Sep 16 13:10:26 node5 kernel: [29355.909073]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909108]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909113]  [<ffffffff8116f100>] ? vm_stat_account+0x0/0x60
Sep 16 13:10:26 node5 kernel: [29355.909147]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909152]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.909157]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.909163]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.909166] /usr/sbin/mun D ffff88178cf8c400     0 170773 166781  736 0x00000004
Sep 16 13:10:26 node5 kernel: [29355.909171]  ffff8806ccf2d5f8 0000000000000086 ffff8806ccf2d5c0 ffff880bd280a000
Sep 16 13:10:26 node5 kernel: [29355.909179]  ffff881f14f2d068 000000000129dfd4 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.909186]  ffff8820737b0800 0000000101ba4c6e ffff88178cf8c9c8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.909194] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.909203]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.909211]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.909233]  [<ffffffffa02d57c7>] buf_hash_find+0x97/0x130 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909257]  [<ffffffffa02dc547>] arc_read+0xb7/0x910 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909280]  [<ffffffffa02dfb64>] ? dbuf_create+0x284/0x3e0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909286]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.909309]  [<ffffffffa02e06e0>] ? dbuf_read_done+0x0/0x110 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909333]  [<ffffffffa02e1360>] dbuf_read+0x1f0/0x770 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909359]  [<ffffffffa02ea269>] dmu_buf_hold_array_by_dnode+0x139/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909386]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909392]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.909426]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909459]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909464]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.909469]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.909502]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909535]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909539]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.909544]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.909548]  [<ffffffff8114d635>] ondemand_readahead+0x115/0x240
Sep 16 13:10:26 node5 kernel: [29355.909553]  [<ffffffff8114d8d6>] page_cache_sync_readahead+0x86/0xa0
Sep 16 13:10:26 node5 kernel: [29355.909559]  [<ffffffff811375fb>] filemap_fault+0x5bb/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.909564]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.909569]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.909574]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.909579]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.909584]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.909589]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.909594]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.909600]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.909605]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.909610]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.909614]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.909619] httpd.itk     S ffff880dec2cd0b0     0 170774  73046 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.909624]  ffff881f00f43e88 0000000000000086 00007ff4cbb58808 ffff8801595a58c0
Sep 16 13:10:26 node5 kernel: [29355.909631]  ffff881ab896e890 0000000000000007 ffff881f00f43f18 ffffffff8104a85c
Sep 16 13:10:26 node5 kernel: [29355.909637]  0000000000000286 ffff881f00f43e50 ffff880dec2cd678 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.909644] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.909648]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.909653]  [<ffffffff8107a917>] do_wait+0x1f7/0x260
Sep 16 13:10:26 node5 kernel: [29355.909657]  [<ffffffff8107aa08>] sys_wait4+0x88/0xd0
Sep 16 13:10:26 node5 kernel: [29355.909662]  [<ffffffff81078c70>] ? child_wait_callback+0x0/0x90
Sep 16 13:10:26 node5 kernel: [29355.909667]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.909673]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.909677] apache2       D ffff883759eb2640     0 170775 1002180 2029 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.909682]  ffff883f543936a8 0000000000000082 ffff883f54393638 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.909689]  393233313230375a ffff8809384988b0 ffff883f54393638 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.909695]  0000000000004030 ffff884071850000 ffff883759eb2c08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.909702] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.909735]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909744]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.909751]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.909774]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909799]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909824]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909848]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909873]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909901]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909906]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.909941]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.909974]  [<ffffffffa0371991>] ? zfs_getpage+0x1/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910007]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910013]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.910018]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.910050]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910082]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910087]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.910092]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.910096]  [<ffffffff8114d635>] ondemand_readahead+0x115/0x240
Sep 16 13:10:26 node5 kernel: [29355.910101]  [<ffffffff8114d8d6>] page_cache_sync_readahead+0x86/0xa0
Sep 16 13:10:26 node5 kernel: [29355.910107]  [<ffffffff811375fb>] filemap_fault+0x5bb/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.910112]  [<ffffffff810a1b55>] ? __wake_up_bit+0x15/0x40
Sep 16 13:10:26 node5 kernel: [29355.910117]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.910122]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.910127]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.910133]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.910138]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.910143]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.910148]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.910153]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.910158]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.910163]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.910168]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.910171] httpd.itk     D ffff880f5a698540     0 170782 170217 1203 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.910177]  ffff8808b57cba98 0000000000000086 ffff8808b57cba60 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.910183]  0000000000000200 000000000199ac5e ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.910190]  ffff8820737b0800 0000000101ba53d0 ffff880f5a698b08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.910197] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.910206]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.910213]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.910236]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910261]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910287]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910311]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910339]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910368]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910394]  [<ffffffffa02ea8b9>] dmu_buf_hold_array+0x39/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910399]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.910425]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910460]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910493]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910498]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.910503]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.910509]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.910513] imap-login    D ffff88022ecd63c0     0 170783 111138 1088 0x00000004
Sep 16 13:10:26 node5 kernel: [29355.910518]  ffff881a483d16f8 0000000000000082 ffff88206dc6c580 0004523000023780
Sep 16 13:10:26 node5 kernel: [29355.910525]  ffff88206dc6c560 ffff88206dc6c550 000000000000000d ffff884071850000
Sep 16 13:10:26 node5 kernel: [29355.910532]  0000000000004030 ffff884070efa3c0 ffff88022ecd6988 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.910538] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.910571]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910580]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.910586]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.910609]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910634]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910658]  [<ffffffffa02e1a4d>] __dbuf_hold_impl+0x16d/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910683]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910706]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910732]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910760]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910765]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.910800]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910833]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910838]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.910843]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.910876]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910909]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.910913]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.910918]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.910923]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.910928]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.910933]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.910939]  [<ffffffff8118c3ea>] ? alloc_pages_current+0xaa/0x110
Sep 16 13:10:26 node5 kernel: [29355.910944]  [<ffffffff8104eb17>] ? pte_alloc_one+0x37/0x50
Sep 16 13:10:26 node5 kernel: [29355.910949]  [<ffffffff811679a2>] ? __pte_alloc+0xd2/0x350
Sep 16 13:10:26 node5 kernel: [29355.910953]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.910958]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.910964]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.910968]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.910972] apache2       D ffff8813b9d430f0     0 170785  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.910977]  ffff8818dce79b38 0000000000000082 0000000000000153 ffff880ef88c0058
Sep 16 13:10:26 node5 kernel: [29355.910984]  00007fab481d9128 ffffffffffffff03 ffffffff8129949f 0000000000000010
Sep 16 13:10:26 node5 kernel: [29355.910991]  0000000000000287 ffff8818dce79ad0 ffff8813b9d436b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.910998] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.911002]  [<ffffffff8129949f>] ? copy_to_user+0x1f/0x30
Sep 16 13:10:26 node5 kernel: [29355.911008]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.911013]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.911018]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.911023]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.911028]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.911033]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.911038]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.911043]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.911047]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.911052]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.911057]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.911062]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.911066]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.911071]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.911076]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.911080]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.911086]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.911089] httpd.itk     S ffff8816c3051130     0 170788 120341 1203 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.911094]  ffff880ef883dcf8 0000000000000082 ffffffff8100beae ffff880ef883dcf8
Sep 16 13:10:26 node5 kernel: [29355.911101]  0000000000000000 ffff881ae1197850 ffff883610d1bb80 ffff880bcffae1a8
Sep 16 13:10:26 node5 kernel: [29355.911108]  0000000000000000 0000000000000000 ffff8816c30516f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.911114] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.911118]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.911125]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.911130]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.911135]  [<ffffffff810a70ea>] ? down_read_trylock+0x1a/0x30
Sep 16 13:10:26 node5 kernel: [29355.911139]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.911145]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.911150]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.911155]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.911160]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.911165]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.911169] flush-zfs-39  S ffff8803833acd70     0 170789      2    0 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.911174]  ffff880a13dfdda0 0000000000000046 0000000000000000 ffff880a13dfdd78
Sep 16 13:10:26 node5 kernel: [29355.911181]  0000000101bb7d2e 0000000000000400 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.911188]  0000000000000000 0000000101bb7da5 ffff8803833ad338 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.911194] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.911199]  [<ffffffff81530902>] schedule_timeout+0x192/0x2e0
Sep 16 13:10:26 node5 kernel: [29355.911205]  [<ffffffff81088830>] ? process_timeout+0x0/0x10
Sep 16 13:10:26 node5 kernel: [29355.911210]  [<ffffffff811dcf3e>] bdi_writeback_task+0x14e/0x1e0
Sep 16 13:10:26 node5 kernel: [29355.911215]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.911219]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.911224]  [<ffffffff81162285>] bdi_start_fn+0x95/0x110
Sep 16 13:10:26 node5 kernel: [29355.911228]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.911233]  [<ffffffff810a15e6>] kthread+0x96/0xa0
Sep 16 13:10:26 node5 kernel: [29355.911238]  [<ffffffff8100c34a>] child_rip+0xa/0x20
Sep 16 13:10:26 node5 kernel: [29355.911243]  [<ffffffff810a1550>] ? kthread+0x0/0xa0
Sep 16 13:10:26 node5 kernel: [29355.911247]  [<ffffffff8100c340>] ? child_rip+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.911250] flush-zfs-33  S ffff8806ccfa7170     0 170790      2    0 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.911255]  ffff880a2e94fda0 0000000000000046 0000000000000000 ffff880a2e94fd78
Sep 16 13:10:26 node5 kernel: [29355.911262]  0000000101bb7d2e 0000000000000400 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.911269]  0000000000000000 0000000101bb7d29 ffff8806ccfa7738 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.911275] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.911280]  [<ffffffff81530902>] schedule_timeout+0x192/0x2e0
Sep 16 13:10:26 node5 kernel: [29355.911286]  [<ffffffff81088830>] ? process_timeout+0x0/0x10
Sep 16 13:10:26 node5 kernel: [29355.911291]  [<ffffffff811dcf3e>] bdi_writeback_task+0x14e/0x1e0
Sep 16 13:10:26 node5 kernel: [29355.911295]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.911300]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.911304]  [<ffffffff81162285>] bdi_start_fn+0x95/0x110
Sep 16 13:10:26 node5 kernel: [29355.911309]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.911314]  [<ffffffff810a15e6>] kthread+0x96/0xa0
Sep 16 13:10:26 node5 kernel: [29355.911318]  [<ffffffff8100c34a>] child_rip+0xa/0x20
Sep 16 13:10:26 node5 kernel: [29355.911323]  [<ffffffff810a1550>] ? kthread+0x0/0xa0
Sep 16 13:10:26 node5 kernel: [29355.911328]  [<ffffffff8100c340>] ? child_rip+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.911331] apache2       D ffff883228d10180     0 170791 279139 2165 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.911336]  ffff88397475db38 0000000000000082 ffff88397475c000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.911343]  0000000000dd99b8 0000000000000002 ffff882073efc470 000280da00000000
Sep 16 13:10:26 node5 kernel: [29355.911349]  0000000000000282 0000000000000030 ffff883228d10748 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.911356] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.911361]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.911365]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.911371]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.911375]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.911380]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.911385]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.911390]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.911394]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.911399]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.911404]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.911408]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.911413]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.911417]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.911422]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.911427]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.911431] couriertls    D ffff88069de4ad30     0 170793  43389  952 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.911436]  ffff880d02d7f358 0000000000000086 ffff880d02d7f320 ffff880d630c7360
Sep 16 13:10:26 node5 kernel: [29355.911443]  0000000000800071 00000000009315d0 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.911449]  ffff8820737b0800 0000000101ba6233 ffff88069de4b2f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.911456] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.911466]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.911472]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.911496]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.911521]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.911525]  [<ffffffff81160a49>] ? zone_statistics+0x99/0xc0
Sep 16 13:10:26 node5 kernel: [29355.911550]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.911574]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.911602]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.911631]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.911656]  [<ffffffffa02eb213>] dmu_read+0x33/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.911682]  [<ffffffffa02eb1e1>] ? dmu_read+0x1/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.911716]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.911750]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.911755]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.911760]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.911792]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.911825]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.911829]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.911834]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.911840]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.911845]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.911850]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.911855]  [<ffffffff8104eb0a>] ? pte_alloc_one+0x2a/0x50
Sep 16 13:10:26 node5 kernel: [29355.911859]  [<ffffffff8104eb17>] ? pte_alloc_one+0x37/0x50
Sep 16 13:10:26 node5 kernel: [29355.911864]  [<ffffffff811679a2>] ? __pte_alloc+0xd2/0x350
Sep 16 13:10:26 node5 kernel: [29355.911869]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.911873]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.911879]  [<ffffffff8117003c>] ? __vma_link_file+0x4c/0x80
Sep 16 13:10:26 node5 kernel: [29355.911884]  [<ffffffff8117014b>] ? vma_link+0x9b/0xf0
Sep 16 13:10:26 node5 kernel: [29355.911888]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.911893]  [<ffffffff8114e385>] ? release_pages+0x255/0x3f0
Sep 16 13:10:26 node5 kernel: [29355.911898]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.911903]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.911908]  [<ffffffff8129adcf>] ? __clear_user+0x3f/0x70
Sep 16 13:10:26 node5 kernel: [29355.911913]  [<ffffffff8129adb1>] ? __clear_user+0x21/0x70
Sep 16 13:10:26 node5 kernel: [29355.911918]  [<ffffffff8129ae38>] clear_user+0x38/0x40
Sep 16 13:10:26 node5 kernel: [29355.911922]  [<ffffffff8120be9d>] padzero+0x2d/0x40
Sep 16 13:10:26 node5 kernel: [29355.911927]  [<ffffffff8120caa7>] load_elf_binary+0x7d7/0x1a10
Sep 16 13:10:26 node5 kernel: [29355.911932]  [<ffffffff81165853>] ? follow_page+0x413/0x500
Sep 16 13:10:26 node5 kernel: [29355.911937]  [<ffffffff8116c6e0>] ? __get_user_pages+0x110/0x410
Sep 16 13:10:26 node5 kernel: [29355.911941]  [<ffffffff8120990f>] ? load_misc_binary+0xaf/0x400
Sep 16 13:10:26 node5 kernel: [29355.911947]  [<ffffffff8129aca6>] ? strnlen_user+0x56/0x80
Sep 16 13:10:26 node5 kernel: [29355.911952]  [<ffffffff811b60f7>] search_binary_handler+0x127/0x370
Sep 16 13:10:26 node5 kernel: [29355.911957]  [<ffffffff811b6aa3>] do_execve+0x223/0x2d0
Sep 16 13:10:26 node5 kernel: [29355.911963]  [<ffffffff8100966a>] sys_execve+0x4a/0x80
Sep 16 13:10:26 node5 kernel: [29355.911967]  [<ffffffff8100b55a>] stub_execve+0x6a/0xc0
Sep 16 13:10:26 node5 kernel: [29355.911971] apache2       D ffff883dbebb91b0     0 170794 1002180 2029 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.911976]  ffff883b1afe9a88 0000000000000086 ffff880000034d88 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.911983]  ffff883b1afe8000 000000000001b720 0000000000000000 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.911990]  ffff882073660b70 000200da00000000 ffff883dbebb9778 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.911997] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.912002]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.912007]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.912012]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.912016]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.912022]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.912027]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.912033]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.912038]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.912043]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.912049]  [<ffffffff81015d24>] ? native_read_tsc+0x4/0x20
Sep 16 13:10:26 node5 kernel: [29355.912054]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.912059]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.912064]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.912069]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.912074]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.912079]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.912084]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.912089]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.912094]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.912099]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.912103]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.912107] apache2       D ffff882ea3c0f1f0     0 170796 279139 2165 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.912112]  ffff88316232db38 0000000000000086 ffff88316232c000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.912119]  000000006232dcc4 0000000000000002 ffffffff8100beae ffff88316232db38
Sep 16 13:10:26 node5 kernel: [29355.912126]  0000000000000246 0000000000000000 ffff882ea3c0f7b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.912132] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.912136]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.912141]  [<ffffffff815324d2>] ? _spin_lock+0x12/0x30
Sep 16 13:10:26 node5 kernel: [29355.912147]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.912152]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.912157]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.912161]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.912166]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.912171]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.912176]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.912181]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.912186]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.912190]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.912195]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.912200]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.912204]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.912209]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.912213]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.912219]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.912223] apache2       D ffff883dbebb86c0     0 170797 1002180 2029 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.912228]  ffff882e0b78fa88 0000000000000082 ffff880000034d88 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.912234]  ffff882e0b78e000 000000000001b720 0000000000000200 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.912241]  ffff882073660b70 000200da00000000 ffff883dbebb8c88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.912248] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.912252]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.912258]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.912262]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.912267]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.912272]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.912277]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.912282]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.912287]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.912293]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.912298]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.912303]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.912308]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.912312]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.912317]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.912323]  [<ffffffff8115bd05>] ? vma_prio_tree_add+0x75/0xd0
Sep 16 13:10:26 node5 kernel: [29355.912328]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.912332]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.912337]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.912342]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.912347]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.912352]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.912356]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.912360] apache2       D ffff882dbb689230     0 170798 1002180 2029 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.912365]  ffff882dbb657a88 0000000000000086 0000000000000000 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.912371]  ffff882dbb656000 000000000001b720 00000000810e39a4 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.912378]  0000000000000000 0000000101ba66f8 ffff882dbb6897f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.912385] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.912389]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.912395]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.912399]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.912404]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.912409]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.912414]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.912419]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.912425]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.912430]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.912435]  [<ffffffff810a1b55>] ? __wake_up_bit+0x15/0x40
Sep 16 13:10:26 node5 kernel: [29355.912439]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.912444]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.912449]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.912454]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.912459]  [<ffffffff8115bd05>] ? vma_prio_tree_add+0x75/0xd0
Sep 16 13:10:26 node5 kernel: [29355.912464]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.912469]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.912474]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.912479]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.912484]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.912488]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.912493]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.912497] sshd          D ffff8806ccfa6680     0 170799  55345 1091 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.912502]  ffff88088a56d6f8 0000000000000086 ffff884073fce280 0004523000000028
Sep 16 13:10:26 node5 kernel: [29355.912508]  ffff884073fce260 ffff884073fce250 0000000000000000 ffff884071850000
Sep 16 13:10:26 node5 kernel: [29355.912515]  0000000000004030 ffff884070efa3c0 ffff8806ccfa6c48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.912522] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.912556]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.912562]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.912566]  [<ffffffff815317d6>] ? down_read+0x16/0x30
Sep 16 13:10:26 node5 kernel: [29355.912571]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.912595]  [<ffffffffa02dee8c>] dbuf_find+0xac/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.912621]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.912645]  [<ffffffffa02e1a4d>] __dbuf_hold_impl+0x16d/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.912670]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.912694]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.912720]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.912725]  [<ffffffff81194d1f>] ? cache_alloc_refill+0x9f/0x230
Sep 16 13:10:26 node5 kernel: [29355.912752]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.912757]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.912792]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.912826]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.912832]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.912837]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.912870]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.912902]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.912907]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.912912]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.912917]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.912922]  [<ffffffff81015d26>] ? native_read_tsc+0x6/0x20
Sep 16 13:10:26 node5 kernel: [29355.912927]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.912932]  [<ffffffff8116af5a>] ? __do_fault+0x30a/0x640
Sep 16 13:10:26 node5 kernel: [29355.912937]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.912942]  [<ffffffff81175e81>] ? page_remove_rmap+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.912947]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.912952]  [<ffffffff8116f236>] ? find_vma+0x46/0x80
Sep 16 13:10:26 node5 kernel: [29355.912957]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.912961]  [<ffffffff811c8168>] ? d_free+0x58/0x60
Sep 16 13:10:26 node5 kernel: [29355.912966]  [<ffffffff811c81c4>] ? d_kill+0x54/0x70
Sep 16 13:10:26 node5 kernel: [29355.912970]  [<ffffffff811d0cd0>] ? mntput_no_expire+0x30/0x110
Sep 16 13:10:26 node5 kernel: [29355.912975]  [<ffffffff811ae82c>] ? __fput+0x1ec/0x280
Sep 16 13:10:26 node5 kernel: [29355.912980]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.912985]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.912989] sshd          D ffff8806ccee11b0     0 170800  55345 1091 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.912994]  ffff8811b343f6f8 0000000000000086 ffff8811b343f6c0 0000000000000007
Sep 16 13:10:26 node5 kernel: [29355.913001]  ffff8836924f8140 000000000056a8cb ffff882100c90770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.913008]  ffff882100c90760 0000000101ba6948 ffff8806ccee1778 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.913014] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.913023]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.913029]  [<ffffffff815317d6>] ? down_read+0x16/0x30
Sep 16 13:10:26 node5 kernel: [29355.913035]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.913058]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913083]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913108]  [<ffffffffa02e1a4d>] __dbuf_hold_impl+0x16d/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913132]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913156]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913182]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913210]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913215]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.913250]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913284]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913289]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.913294]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.913327]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913359]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913364]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.913369]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.913374]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.913379]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.913385]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.913390]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.913395]  [<ffffffff811ccd5d>] ? notify_change+0x2bd/0x340
Sep 16 13:10:26 node5 kernel: [29355.913400]  [<ffffffff81175e81>] ? page_remove_rmap+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.913405]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.913409]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.913414]  [<ffffffff811c8168>] ? d_free+0x58/0x60
Sep 16 13:10:26 node5 kernel: [29355.913418]  [<ffffffff811c81c4>] ? d_kill+0x54/0x70
Sep 16 13:10:26 node5 kernel: [29355.913422]  [<ffffffff811d0cd0>] ? mntput_no_expire+0x30/0x110
Sep 16 13:10:26 node5 kernel: [29355.913428]  [<ffffffff811ae82c>] ? __fput+0x1ec/0x280
Sep 16 13:10:26 node5 kernel: [29355.913433]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.913437]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.913441] apache2       D ffff8805c49d4500     0 170801  99398 3205 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.913446]  ffff8811b34b16a8 0000000000000082 0000000000000000 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.913453]  0000000000000000 ffff8811b34816c0 ffff8811b34b1638 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.913460]  0000000000000000 0000000101ba6a1c ffff8805c49d4ac8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.913466] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.913498]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913508]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.913514]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.913538]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913563]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913588]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913612]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913638]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913665]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913670]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.913705]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913738]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913743]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.913748]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.913780]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913812]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913817]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.913822]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.913826]  [<ffffffff8114d635>] ondemand_readahead+0x115/0x240
Sep 16 13:10:26 node5 kernel: [29355.913831]  [<ffffffff8114d8d6>] page_cache_sync_readahead+0x86/0xa0
Sep 16 13:10:26 node5 kernel: [29355.913836]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.913841]  [<ffffffff811375fb>] filemap_fault+0x5bb/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.913846]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.913851]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.913856]  [<ffffffff81291c3b>] ? prio_tree_insert+0x26b/0x2b0
Sep 16 13:10:26 node5 kernel: [29355.913861]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.913866]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.913871]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.913876]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.913881]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.913886]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.913891]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.913895] apache2       D ffff883daaef0500     0 170813 145855 2028 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.913901]  ffff883780eb35f8 0000000000000082 0000000000000000 ffff88178f6f0000
Sep 16 13:10:26 node5 kernel: [29355.913907]  ffff88200b8f0a40 000000000000000d ffff88012bed0770 ffff8820720b78c0
Sep 16 13:10:26 node5 kernel: [29355.913914]  ffff88012bed0760 0000000101ba75d3 ffff883daaef0ac8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.913921] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.913930]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.913939]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.913961]  [<ffffffffa02d57c7>] buf_hash_find+0x97/0x130 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.913984]  [<ffffffffa02dc547>] arc_read+0xb7/0x910 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914008]  [<ffffffffa02dfb64>] ? dbuf_create+0x284/0x3e0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914013]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.914037]  [<ffffffffa02e06e0>] ? dbuf_read_done+0x0/0x110 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914061]  [<ffffffffa02e1360>] dbuf_read+0x1f0/0x770 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914087]  [<ffffffffa02ea269>] dmu_buf_hold_array_by_dnode+0x139/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914114]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914120]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.914154]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914187]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914192]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.914197]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.914229]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914261]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914266]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.914271]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.914275]  [<ffffffff8114d635>] ondemand_readahead+0x115/0x240
Sep 16 13:10:26 node5 kernel: [29355.914280]  [<ffffffff8114d8d6>] page_cache_sync_readahead+0x86/0xa0
Sep 16 13:10:26 node5 kernel: [29355.914285]  [<ffffffff811375fb>] filemap_fault+0x5bb/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.914291]  [<ffffffff810a1b55>] ? __wake_up_bit+0x15/0x40
Sep 16 13:10:26 node5 kernel: [29355.914296]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.914301]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.914306]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.914311]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.914317]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.914321]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.914326]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.914331]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.914336]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.914342]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.914346]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.914350] apache2       D ffff882ea3c0e700     0 170814 279139 2165 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.914355]  ffff8838f0679b38 0000000000000082 ffff8838f0679b00 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.914362]  0000000000000002 00000000003b3fda ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.914369]  ffff8820737b0800 0000000101bb1e25 ffff882ea3c0ecc8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.914375] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.914381]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.914385]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.914390]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.914395]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.914400]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.914404]  [<ffffffff81194b53>] ? cache_grow+0x1f3/0x320
Sep 16 13:10:26 node5 kernel: [29355.914409]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.914414]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.914419]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.914424]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.914429]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.914433]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.914438]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.914442]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.914447]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.914452]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.914456] apache2       D ffff881fa07112b0     0 170815  50154  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.914461]  ffff880476bb1b38 0000000000000086 0000000000000000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.914468]  ffffffff8100beae ffff880476bb1b48 00000000000003e1 ffff880429f6b840
Sep 16 13:10:26 node5 kernel: [29355.914474]  0000000000000000 0000000101bac345 ffff881fa0711878 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.914481] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.914485]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.914490]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.914496]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.914500]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.914506]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.914510]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.914515]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.914520]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.914525]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.914530]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.914534]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.914539]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.914544]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.914548]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.914553]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.914557]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.914562]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.914567]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.914571] /usr/sbin/mun ? ffff8819ee041330     0 170818 166781  736 0x00000004
Sep 16 13:10:26 node5 kernel: [29355.914576]  ffff8805dc647cd8 0000000000000046 0000000000000011 ffff881900040002
Sep 16 13:10:26 node5 kernel: [29355.914582]  0000000000004b13 ffffffff00000001 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.914589]  0000000000000000 0000000000000000 ffff8819ee0418f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.914595] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.914600]  [<ffffffff810a7744>] ? switch_task_namespaces+0x24/0x60
Sep 16 13:10:26 node5 kernel: [29355.914605]  [<ffffffff810a7720>] ? switch_task_namespaces+0x0/0x60
Sep 16 13:10:26 node5 kernel: [29355.914610]  [<ffffffff8107b255>] do_exit+0x635/0x960
Sep 16 13:10:26 node5 kernel: [29355.914614]  [<ffffffff81291ae3>] ? prio_tree_insert+0x113/0x2b0
Sep 16 13:10:26 node5 kernel: [29355.914618]  [<ffffffff8107b5d8>] do_group_exit+0x58/0xd0
Sep 16 13:10:26 node5 kernel: [29355.914624]  [<ffffffff810920b6>] get_signal_to_deliver+0x1f6/0x460
Sep 16 13:10:26 node5 kernel: [29355.914629]  [<ffffffff8100a2c5>] do_signal+0x75/0x800
Sep 16 13:10:26 node5 kernel: [29355.914634]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.914639]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.914644]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.914649]  [<ffffffff8100aae0>] do_notify_resume+0x90/0xc0
Sep 16 13:10:26 node5 kernel: [29355.914654]  [<ffffffff8100bb9c>] retint_signal+0x48/0x8c
Sep 16 13:10:26 node5 kernel: [29355.914658] exim          D ffff880d810d6800     0 170819 148581 1297 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.914663]  ffff881a6b6f32c8 0000000000000082 ffffffff8100beae ffff881a6b6f32b8
Sep 16 13:10:26 node5 kernel: [29355.914669]  0000000000000000 000000000000001b 000000000000001c ffff8812c1de7000
Sep 16 13:10:26 node5 kernel: [29355.914676]  0000000000000000 0000000000000023 ffff880d810d6dc8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.914682] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.914686]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.914696]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.914705]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.914726]  [<ffffffffa02d57c7>] buf_hash_find+0x97/0x130 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914749]  [<ffffffffa02dc547>] arc_read+0xb7/0x910 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914773]  [<ffffffffa02dfb80>] ? dbuf_create+0x2a0/0x3e0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914797]  [<ffffffffa02e06e0>] ? dbuf_read_done+0x0/0x110 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914821]  [<ffffffffa02e1360>] dbuf_read+0x1f0/0x770 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914847]  [<ffffffffa02ea269>] dmu_buf_hold_array_by_dnode+0x139/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914874]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914880]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.914915]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914949]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.914954]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.914959]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.914992]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915024]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915029]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.915034]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.915039]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.915044]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.915049]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.915055]  [<ffffffff8118c3ea>] ? alloc_pages_current+0xaa/0x110
Sep 16 13:10:26 node5 kernel: [29355.915059]  [<ffffffff8104eb17>] ? pte_alloc_one+0x37/0x50
Sep 16 13:10:26 node5 kernel: [29355.915064]  [<ffffffff811679a2>] ? __pte_alloc+0xd2/0x350
Sep 16 13:10:26 node5 kernel: [29355.915069]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.915074]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.915079]  [<ffffffff8117003c>] ? __vma_link_file+0x4c/0x80
Sep 16 13:10:26 node5 kernel: [29355.915084]  [<ffffffff8117014b>] ? vma_link+0x9b/0xf0
Sep 16 13:10:26 node5 kernel: [29355.915089]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.915094]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.915099]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.915104]  [<ffffffff8129adcf>] ? __clear_user+0x3f/0x70
Sep 16 13:10:26 node5 kernel: [29355.915109]  [<ffffffff8129adb1>] ? __clear_user+0x21/0x70
Sep 16 13:10:26 node5 kernel: [29355.915114]  [<ffffffff8129ae38>] clear_user+0x38/0x40
Sep 16 13:10:26 node5 kernel: [29355.915118]  [<ffffffff8120bdb0>] ? set_brk+0x0/0xc0
Sep 16 13:10:26 node5 kernel: [29355.915122]  [<ffffffff8120be9d>] padzero+0x2d/0x40
Sep 16 13:10:26 node5 kernel: [29355.915127]  [<ffffffff8120caa7>] load_elf_binary+0x7d7/0x1a10
Sep 16 13:10:26 node5 kernel: [29355.915132]  [<ffffffff81165853>] ? follow_page+0x413/0x500
Sep 16 13:10:26 node5 kernel: [29355.915137]  [<ffffffff8116c6e0>] ? __get_user_pages+0x110/0x410
Sep 16 13:10:26 node5 kernel: [29355.915141]  [<ffffffff8120990f>] ? load_misc_binary+0xaf/0x400
Sep 16 13:10:26 node5 kernel: [29355.915147]  [<ffffffff811b54fe>] ? copy_strings+0x9e/0x230
Sep 16 13:10:26 node5 kernel: [29355.915152]  [<ffffffff811b60f7>] search_binary_handler+0x127/0x370
Sep 16 13:10:26 node5 kernel: [29355.915157]  [<ffffffff811b6aa3>] do_execve+0x223/0x2d0
Sep 16 13:10:26 node5 kernel: [29355.915162]  [<ffffffff8100966a>] sys_execve+0x4a/0x80
Sep 16 13:10:26 node5 kernel: [29355.915167]  [<ffffffff8100b55a>] stub_execve+0x6a/0xc0
Sep 16 13:10:26 node5 kernel: [29355.915171] apache2       D ffff88337e34b270     0 170820 279139 2165 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.915176]  ffff883792759618 0000000000000082 ffff8837927595e0 0000000000000286
Sep 16 13:10:26 node5 kernel: [29355.915182]  ffff883792759598 0000000000d54356 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.915189]  ffff8820737b0800 0000000101ba8196 ffff88337e34b838 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.915196] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.915205]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.915212]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.915235]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915260]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915285]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915309]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915334]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915360]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915394]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915429]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915463]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915497]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915530]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915562]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915568]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.915602]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915637]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.915641]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.915646]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.915652]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.915657]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.915661]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.915666]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.915671]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.915676]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.915682]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.915687]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.915692]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.915697]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.915702]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.915706]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.915712]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.915716] apache2       D ffff88337e34a780     0 170821 279139 2165 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.915721]  ffff883e1cb5fb38 0000000000000086 ffff883e1cb5fb00 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.915728]  000000001cb5fad8 0000000001ae3307 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.915734]  ffff8820737b0800 0000000101ba85b5 ffff88337e34ad48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.915741] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.915746]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.915751]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.915756]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.915761]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.915766]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.915771]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.915776]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.915781]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.915790]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.915795]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.915799]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.915805]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.915810]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.915815]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.915820]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.915824]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.915829]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.915834]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.915838] apache2       D ffff88326dd672b0     0 170822 279139 2165 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.915843]  ffff88337ab73b38 0000000000000082 ffff88337ab72000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.915850]  000000007ab73b28 0000000000000002 ffff882073672c70 000280da00000000
Sep 16 13:10:26 node5 kernel: [29355.915856]  0000000000000282 0000000000000030 ffff88326dd67878 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.915863] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.915868]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.915872]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.915878]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.915882]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.915887]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.915892]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.915897]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.915901]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.915906]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.915911]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.915916]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.915921]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.915926]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.915931]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.915935]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.915940]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.915945]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.915950]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.915954] apache2       D ffff881055ed4700     0 170824  79040 1187 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.915959]  ffff881879e37638 0000000000000086 ffff880142030440 ffff88207049f930
Sep 16 13:10:26 node5 kernel: [29355.915966]  ffff881879e37668 ffffffffa02f9ae6 ffffffff8100beae ffff881879e37638
Sep 16 13:10:26 node5 kernel: [29355.915973]  0000000000000000 ffff881879e377b0 ffff881055ed4cc8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.915980] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.916008]  [<ffffffffa02f9ae6>] ? dmu_zfetch+0x656/0xe30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916013]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.916019]  [<ffffffff81055a18>] ? mutex_spin_on_owner+0xb8/0xc0
Sep 16 13:10:26 node5 kernel: [29355.916028]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.916037]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.916060]  [<ffffffffa02d75fa>] arc_buf_add_ref+0x9a/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916084]  [<ffffffffa02e1946>] __dbuf_hold_impl+0x66/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916108]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916132]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916158]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916184]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916218]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916252]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916285]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916318]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916350]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916383]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916388]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.916422]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916455]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.916460]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.916466]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.916471]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.916476]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.916480]  [<ffffffff811ba76b>] ? get_fs_root+0x2b/0x80
Sep 16 13:10:26 node5 kernel: [29355.916485]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.916490]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.916495]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.916500]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.916505]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.916511]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.916515]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.916520]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.916525]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.916530]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.916535]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.916539] apache2       D ffff882dca0db2f0     0 170825 145855 2028 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.916544]  ffff883fcd42ba88 0000000000000086 ffff880000034d88 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.916550]  ffff883fcd42a000 000000000001b720 0000000000000000 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.916557]  ffff882073672c70 000200da00000000 ffff882dca0db8b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.916564] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.916569]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.916574]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.916579]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.916584]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.916589]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.916594]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.916600]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.916605]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.916610]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.916615]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.916620]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.916625]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.916630]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.916636]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.916640]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.916645]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.916651]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.916656]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.916661]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.916665]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.916669] apache2       D ffff8810b85731b0     0 170840  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.916674]  ffff8804c4a8db38 0000000000000086 ffff8804c4a8c000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.916681]  00000000c4a8dcc4 0000000000000002 ffff88207369fdf0 000280da00000000
Sep 16 13:10:26 node5 kernel: [29355.916688]  0000000000000286 0000000000000030 ffff8810b8573778 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.916696] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.916702]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.916706]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.916712]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.916716]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.916720]  [<ffffffff811ba8db>] ? acl_permission_check+0x3b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.916725]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.916731]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.916736]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.916740]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.916745]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.916750]  [<ffffffff8116f250>] ? find_vma+0x60/0x80
Sep 16 13:10:26 node5 kernel: [29355.916754]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.916759]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.916764]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.916768]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.916773]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.916777]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.916783]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.916787] apache2       D ffff882ec42eccf0     0 170841 279139 2165 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.916792]  ffff8833fe6e3b38 0000000000000082 ffff8833fe6e2000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.916799]  00000000fe6e3ad8 0000000000000002 ffff884073428bf0 000280da00000000
Sep 16 13:10:26 node5 kernel: [29355.916805]  0000000000000286 0000000000000030 ffff882ec42ed2b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.916812] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.916817]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.916821]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.916826]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.916831]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.916838]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.916843]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.916847]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.916852]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.916857]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.916862]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.916866]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.916871]  [<ffffffff810a70ea>] ? down_read_trylock+0x1a/0x30
Sep 16 13:10:26 node5 kernel: [29355.916877]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.916882]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.916887]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.916892]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.916896]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.916901]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.916906]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.916910] flush-zfs-61  S ffff8803cea95430     0 170842      2    0 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.916915]  ffff881935961da0 0000000000000046 0000000000000000 ffff881935961d78
Sep 16 13:10:26 node5 kernel: [29355.916922]  0000000101bb7d36 0000000000000400 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.916928]  0000000000000000 0000000000000024 ffff8803cea959f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.916935] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.916940]  [<ffffffff81530902>] schedule_timeout+0x192/0x2e0
Sep 16 13:10:26 node5 kernel: [29355.916946]  [<ffffffff81088830>] ? process_timeout+0x0/0x10
Sep 16 13:10:26 node5 kernel: [29355.916951]  [<ffffffff811dcf3e>] bdi_writeback_task+0x14e/0x1e0
Sep 16 13:10:26 node5 kernel: [29355.916956]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.916960]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.916964]  [<ffffffff81162285>] bdi_start_fn+0x95/0x110
Sep 16 13:10:26 node5 kernel: [29355.916969]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.916974]  [<ffffffff810a15e6>] kthread+0x96/0xa0
Sep 16 13:10:26 node5 kernel: [29355.916978]  [<ffffffff8100c34a>] child_rip+0xa/0x20
Sep 16 13:10:26 node5 kernel: [29355.916983]  [<ffffffff810a1550>] ? kthread+0x0/0xa0
Sep 16 13:10:26 node5 kernel: [29355.916988]  [<ffffffff8100c340>] ? child_rip+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.916991] apache2       D ffff8813b9d42600     0 170843  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.916996]  ffff881aaac07b38 0000000000000086 ffff881aaac07b00 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.917003]  0000000000e199b8 0000000001076cb0 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.917010]  ffff8820737b0800 0000000101bb16f0 ffff8813b9d42bc8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.917016] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.917021]  [<ffffffff81055979>] ? mutex_spin_on_owner+0x19/0xc0
Sep 16 13:10:26 node5 kernel: [29355.917026]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.917031]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.917036]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.917041]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.917045]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.917051]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.917056]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.917060]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.917065]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.917070]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.917074]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.917079]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.917084]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.917088]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.917093]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.917098]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.917102] apache2       D ffff88047cf211f0     0 170844  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.917107]  ffff8810404b3b38 0000000000000082 0000000000000000 ffff8831e79d12f8
Sep 16 13:10:26 node5 kernel: [29355.917114]  ffff8810404b3b08 ffffffffa020b37f 0000000000000069 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.917120]  0000000000000000 ffffffffa0396d5d ffff88047cf217b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.917127] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.917139]  [<ffffffffa020b37f>] ? tsd_exit+0x5f/0x2b0 [spl]
Sep 16 13:10:26 node5 kernel: [29355.917145]  [<ffffffff811c766e>] ? __d_instantiate+0xbe/0x110
Sep 16 13:10:26 node5 kernel: [29355.917151]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.917156]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.917161]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.917166]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.917171]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.917176]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.917180]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.917186]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.917191]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.917196]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.917201]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.917206]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.917211]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.917216]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.917220]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.917225]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.917229] apache2       D ffff8803cea94940     0 170845  25957  283 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.917234]  ffff881491c97490 0000000000000082 ffff881491c97458 00007ff604006d72
Sep 16 13:10:26 node5 kernel: [29355.917241]  0000000000000001 0000000000b7da3d ffff8840734c2410 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.917248]  ffff8840734c2400 0000000101ba9525 ffff8803cea94f08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.917254] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.917259]  [<ffffffff81532155>] rwsem_down_failed_common+0x95/0x1d0
Sep 16 13:10:26 node5 kernel: [29355.917263]  [<ffffffff815322e6>] rwsem_down_read_failed+0x26/0x30
Sep 16 13:10:26 node5 kernel: [29355.917269]  [<ffffffff8129a9d4>] call_rwsem_down_read_failed+0x14/0x30
Sep 16 13:10:26 node5 kernel: [29355.917273]  [<ffffffff815317e4>] ? down_read+0x24/0x30
Sep 16 13:10:26 node5 kernel: [29355.917302]  [<ffffffffa02f9549>] dmu_zfetch+0xb9/0xe30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917327]  [<ffffffffa02e1861>] dbuf_read+0x6f1/0x770 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917355]  [<ffffffffa02fc879>] dnode_hold_impl+0x129/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917383]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917409]  [<ffffffffa02e969a>] dmu_buf_hold_noread+0x3a/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917414]  [<ffffffff8114e880>] ? __lru_cache_add+0x40/0x90
Sep 16 13:10:26 node5 kernel: [29355.917439]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917445]  [<ffffffff811350b7>] ? unlock_page+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.917480]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917485]  [<ffffffff81142773>] ? __rmqueue+0xc3/0x490
Sep 16 13:10:26 node5 kernel: [29355.917490]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.917525]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917559]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917593]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917627]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917660]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917665]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.917670]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.917705]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917739]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917744]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.917749]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.917755]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.917759]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.917764]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.917769]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.917774]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.917779]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.917784]  [<ffffffff8116f240>] ? find_vma+0x50/0x80
Sep 16 13:10:26 node5 kernel: [29355.917789]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.917794]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.917799]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.917804]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.917809]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.917814]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.917819]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.917823] apache2       D ffff882dca0da800     0 170846  98640 1223 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.917828]  ffff88374e3035f8 0000000000000082 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.917834]  ffff8820720b7900 0004523000000282 ffff88012bed0770 ffff8820720b78d0
Sep 16 13:10:26 node5 kernel: [29355.917841]  ffff88012bed0760 0000000101ba9629 ffff882dca0dadc8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.917848] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.917857]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.917865]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.917886]  [<ffffffffa02d57c7>] buf_hash_find+0x97/0x130 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917910]  [<ffffffffa02dc547>] arc_read+0xb7/0x910 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917934]  [<ffffffffa02dfb64>] ? dbuf_create+0x284/0x3e0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917939]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.917962]  [<ffffffffa02e06e0>] ? dbuf_read_done+0x0/0x110 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.917986]  [<ffffffffa02e1360>] dbuf_read+0x1f0/0x770 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918013]  [<ffffffffa02ea269>] dmu_buf_hold_array_by_dnode+0x139/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918040]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918046]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.918081]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918116]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918121]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.918126]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.918160]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918193]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918198]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.918203]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.918208]  [<ffffffff8114d635>] ondemand_readahead+0x115/0x240
Sep 16 13:10:26 node5 kernel: [29355.918212]  [<ffffffff8114d8d6>] page_cache_sync_readahead+0x86/0xa0
Sep 16 13:10:26 node5 kernel: [29355.918218]  [<ffffffff811375fb>] filemap_fault+0x5bb/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.918223]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.918228]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.918233]  [<ffffffff81291ab2>] ? prio_tree_insert+0xe2/0x2b0
Sep 16 13:10:26 node5 kernel: [29355.918238]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.918243]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.918248]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.918253]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.918258]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.918263]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.918268]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.918272] apache2       D ffff8817d9a34d70     0 170850  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.918277]  ffff88071227bb38 0000000000000086 0000000000000000 000000000000000e
Sep 16 13:10:26 node5 kernel: [29355.918284]  ffff88071227bb28 ffffffff8114e385 ffffffff8100beae ffff88071227bb48
Sep 16 13:10:26 node5 kernel: [29355.918290]  0000000000000000 0000000101bb393a ffff8817d9a35338 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.918297] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.918301]  [<ffffffff8114e385>] ? release_pages+0x255/0x3f0
Sep 16 13:10:26 node5 kernel: [29355.918306]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.918312]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.918317]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.918322]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.918326]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.918332]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.918336]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.918341]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.918346]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.918350]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.918355]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.918360]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.918364]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.918369]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.918373]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.918379]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.918383] apache2       D ffff8817d9a34280     0 170851  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.918388]  ffff881879ff7618 0000000000000086 ffff881879ff75e0 ffffffffffffff03
Sep 16 13:10:26 node5 kernel: [29355.918394]  ffff881879ff7598 0000000000812b40 ffff88012be50770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.918401]  ffff88012be50760 0000000101bb4065 ffff8817d9a34848 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.918408] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.918417]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.918424]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.918448]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918473]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918497]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918521]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918546]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918572]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918606]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918635]  [<ffffffffa02fa8d4>] ? dnode_rele+0x54/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918640]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.918645]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.918680]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918713]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918746]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918779]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918812]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918817]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.918851]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918885]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.918890]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.918895]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.918901]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.918906]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.918910]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.918915]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.918920]  [<ffffffff8116f211>] ? find_vma+0x21/0x80
Sep 16 13:10:26 node5 kernel: [29355.918925]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.918930]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.918935]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.918939]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.918944]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.918949]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.918954]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.918958] apache2       D ffff8811f975cff0     0 170852  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.918963]  ffff8809cd599b38 0000000000000082 ffff8809cd599b00 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.918970]  0000000000dd99b8 00000000032b3684 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.918977]  ffff8820737b0800 0000000101bb6b25 ffff8811f975d5b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.918984] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.918989]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.918994]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.918999]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.919004]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.919008]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.919014]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.919019]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.919023]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.919028]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.919033]  [<ffffffff8116f236>] ? find_vma+0x46/0x80
Sep 16 13:10:26 node5 kernel: [29355.919037]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.919044]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.919048]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.919053]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.919058]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.919062]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.919067]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.919071] apache2       D ffff88025e9734b0     0 170853  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.919076]  ffff8808f308db38 0000000000000086 0000000000000000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.919082]  0000000000000200 0000000000000002 ffff8820737b0810 000280da00000000
Sep 16 13:10:26 node5 kernel: [29355.919089]  ffff8820737b0800 0000000101bb6e5f ffff88025e973a78 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.919096] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.919101]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.919105]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.919111]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.919115]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.919120]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.919125]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.919130]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.919134]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.919139]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.919143]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.919148]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.919153]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.919157]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.919162]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.919167]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.919172]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.919176] couriertcpd   D ffff881fa07107c0     0 170854 334471 1193 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.919181]  ffff881ce74f1a88 0000000000000082 0000000000000000 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.919188]  ffff881ce74f0000 000000000001b720 ffff8840734c2410 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.919195]  ffff8840734c2400 0000000101ba9836 ffff881fa0710d88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.919201] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.919206]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.919212]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.919216]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.919221]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.919226]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.919231]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.919237]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.919242]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.919247]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.919252]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.919258]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.919263]  [<ffffffff8118c3ea>] ? alloc_pages_current+0xaa/0x110
Sep 16 13:10:26 node5 kernel: [29355.919268]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.919273]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.919277]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.919283]  [<ffffffff81454193>] ? move_addr_to_user+0x93/0xb0
Sep 16 13:10:26 node5 kernel: [29355.919288]  [<ffffffff81454814>] ? sys_getsockname+0xd4/0xf0
Sep 16 13:10:26 node5 kernel: [29355.919293]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.919298]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.919301] apache2       D ffff881613a10900     0 170855  99398 3205 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.919306]  ffff880dced03a88 0000000000000086 ffff882080022c48 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.919313]  ffff880dced02000 000000000001b720 0000000000000200 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.919320]  ffff88407346f2f0 000200da00000000 ffff881613a10ec8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.919326] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.919331]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.919336]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.919341]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.919346]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.919351]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.919355]  [<ffffffff811caf60>] ? inode_init_always+0x100/0x1b0
Sep 16 13:10:26 node5 kernel: [29355.919361]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.919366]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.919371]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.919376]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.919381]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.919387]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.919391]  [<ffffffff812919d6>] ? prio_tree_insert+0x6/0x2b0
Sep 16 13:10:26 node5 kernel: [29355.919396]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.919401]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.919406]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.919411]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.919416]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.919421]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.919425]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.919429] apache2       D ffff882dbfd5d330     0 170856  98640 1223 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.919434]  ffff8832027d9a88 0000000000000086 ffff880000034d88 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.919441]  ffff8832027d8000 000000000001b720 0000000000000000 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.919447]  ffff882073660b70 000200da00000000 ffff882dbfd5d8f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.919454] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.919458]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.919464]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.919468]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.919473]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.919478]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.919483]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.919489]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.919494]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.919499]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.919504]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.919509]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.919514]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.919519]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.919524]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.919529]  [<ffffffff8116f236>] ? find_vma+0x46/0x80
Sep 16 13:10:26 node5 kernel: [29355.919533]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.919538]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.919543]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.919548]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.919553]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.919557]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.919561] apache2       D ffff882dbfd5c840     0 170857  98640 1223 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.919566]  ffff88305e489a88 0000000000000082 0000000000000000 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.919573]  ffff88305e488000 0000000000582f4f ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.919579]  0000000000000000 0000000101ba9a15 ffff882dbfd5ce08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.919586] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.919591]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.919596]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.919601]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.919605]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.919610]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.919615]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.919621]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.919626]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.919631]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.919636]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.919641]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.919645]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.919650]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.919655]  [<ffffffff8115bd05>] ? vma_prio_tree_add+0x75/0xd0
Sep 16 13:10:26 node5 kernel: [29355.919660]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.919665]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.919670]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.919675]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.919680]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.919684]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.919689]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.919692] apache2       D ffff88025e9729c0     0 170858  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.919697]  ffff881c695b7b38 0000000000000082 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.919704]  ffff88407357a6d0 ffff88015f8435c0 ffffffff8100beae ffff881c695b7b38
Sep 16 13:10:26 node5 kernel: [29355.919711]  0000000000000293 0000000000000000 ffff88025e972f88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.919717] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.919721]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.919726]  [<ffffffff810559be>] ? mutex_spin_on_owner+0x5e/0xc0
Sep 16 13:10:26 node5 kernel: [29355.919731]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.919736]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.919741]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.919745]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.919750]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.919755]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.919760]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.919765]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.919770]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.919775]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.919780]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.919785]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.919790]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.919794]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.919799]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.919804]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.919809]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.919812] apache2       D ffff88169a5fab30     0 170859  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.919817]  ffff881672f7b618 0000000000000086 ffff881672f7b5e0 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.919824]  ffff881672f7b598 000000000321e952 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.919831]  ffff8820737b0800 0000000101bb76ae ffff88169a5fb0f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.919837] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.919842]  [<ffffffff81055979>] ? mutex_spin_on_owner+0x19/0xc0
Sep 16 13:10:26 node5 kernel: [29355.919851]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.919859]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.919883]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.919908]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.919932]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.919956]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.919982]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920008]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920043]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920048]  [<ffffffff81142773>] ? __rmqueue+0xc3/0x490
Sep 16 13:10:26 node5 kernel: [29355.920083]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920088]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.920122]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920156]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920190]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920226]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920261]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920266]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.920271]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.920276]  [<ffffffff811bb990>] ? path_init+0x120/0x220
Sep 16 13:10:26 node5 kernel: [29355.920281]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.920286]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.920291]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.920296]  [<ffffffff8116f236>] ? find_vma+0x46/0x80
Sep 16 13:10:26 node5 kernel: [29355.920300]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.920307]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.920312]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.920316]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.920321]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.920326]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.920331]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.920335] apache2       D ffff88169a5fa040     0 170860  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.920340]  ffff88089be85678 0000000000000086 00000000000003ae 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.920347]  0000000000000000 0000000000000000 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.920353]  0000000000000000 0000000000000000 ffff88169a5fa608 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.920360] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.920383]  [<ffffffffa02defc6>] ? RW_WRITE_HELD+0x66/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920388]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.920397]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.920405]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.920428]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920453]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920477]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920501]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920530]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920554]  [<ffffffffa02e0149>] ? dbuf_rele_and_unlock+0x229/0x320 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920582]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920608]  [<ffffffffa02e9984>] dmu_bonus_hold+0x34/0x290 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920631]  [<ffffffffa02e0640>] ? dbuf_rele+0x30/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920664]  [<ffffffffa031ecde>] sa_buf_hold+0xe/0x10 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920699]  [<ffffffffa0376d9f>] zfs_zget+0x10f/0x1e0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920733]  [<ffffffffa034daae>] ? zap_lookup_norm+0xbe/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920766]  [<ffffffffa0356ccc>] zfs_dirent_lock+0x47c/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920799]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920832]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920864]  [<ffffffffa0350e61>] ? zfs_zaccess+0x1/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920898]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920933]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.920938]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.920943]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.920948]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.920953]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.920958]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.920963]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.920968]  [<ffffffff8116f236>] ? find_vma+0x46/0x80
Sep 16 13:10:26 node5 kernel: [29355.920973]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.920978]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.920983]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.920987]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.920992]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.920997]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.921002]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.921006] apache2       D ffff8804c81d0b70     0 170861  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.921011]  ffff8818f3c9fb38 0000000000000082 ffff8818f3c9fb00 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.921018]  0000000000000002 000000000175f853 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.921025]  ffff8820737b0800 0000000101bb7e5d ffff8804c81d1138 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.921031] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.921037]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.921041]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.921047]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.921051]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.921056]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.921060]  [<ffffffff81194b53>] ? cache_grow+0x1f3/0x320
Sep 16 13:10:26 node5 kernel: [29355.921065]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.921070]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.921075]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.921079]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.921084]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.921089]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.921093]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.921098]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.921102]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.921108]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.921112] apache2       D ffff8804c81d0080     0 170862  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.921117]  ffff880a54bcbb38 0000000000000086 ffff880a54bcbb00 000000000000000e
Sep 16 13:10:26 node5 kernel: [29355.921124]  ffff880a54bcbb28 0000000000fbba94 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.921130]  ffff8820737b0800 0000000101bb8343 ffff8804c81d0648 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.921137] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.921142]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.921146]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.921152]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.921156]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.921161]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.921166]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.921171]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.921176]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.921181]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.921186]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.921191]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.921196]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.921201]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.921206]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.921211]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.921215]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.921220]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.921224] apache2       S ffff880490cdabb0     0 170863  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.921229]  ffff881972c5fda8 0000000000000082 ffff881972c5fd70 ffffffff810a6ad3
Sep 16 13:10:26 node5 kernel: [29355.921236]  ffff881900000013 0000000000ffa967 ffff882100cd0770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.921243]  ffff882100cd0760 0000000101bb8342 ffff880490cdb178 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.921250] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.921254]  [<ffffffff810a6ad3>] ? __hrtimer_start_range_ns+0x1a3/0x460
Sep 16 13:10:26 node5 kernel: [29355.921260]  [<ffffffff81531578>] schedule_hrtimeout_range+0xc8/0x160
Sep 16 13:10:26 node5 kernel: [29355.921265]  [<ffffffff810a5dc0>] ? hrtimer_wakeup+0x0/0x30
Sep 16 13:10:26 node5 kernel: [29355.921270]  [<ffffffff810a6dc4>] ? hrtimer_start_range_ns+0x14/0x20
Sep 16 13:10:26 node5 kernel: [29355.921275]  [<ffffffff811f91ac>] ep_poll+0x2bc/0x350
Sep 16 13:10:26 node5 kernel: [29355.921280]  [<ffffffff81060020>] ? default_wake_function+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.921286]  [<ffffffff811f9305>] sys_epoll_wait+0xc5/0xe0
Sep 16 13:10:26 node5 kernel: [29355.921291]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.921295] apache2       S ffff880490cda0c0     0 170864  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.921300]  ffff880bb657fcf8 0000000000000086 ffff880bb657fcc0 00007fab4572c250
Sep 16 13:10:26 node5 kernel: [29355.921307]  ffff880bb657fc78 00000000008e1c40 ffff882100c10770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.921314]  ffff882100c10760 0000000101ba9b9b ffff880490cda688 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.921320] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.921325]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.921331]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.921336]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.921342]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.921347]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.921352]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.921357]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.921362]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.921366] apache2       S ffff88016c28cbf0     0 170865  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.921371]  ffff88088a47bcf8 0000000000000082 ffff881b31a26960 00007fab4572c250
Sep 16 13:10:26 node5 kernel: [29355.921378]  ffff88088a47bc78 ffffffff81193a10 ffff88205a9e0980 ffff88207fc07380
Sep 16 13:10:26 node5 kernel: [29355.921384]  ffff88088a47bca8 ffffffff810b764c ffff88016c28d1b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.921391] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.921395]  [<ffffffff81193a10>] ? ub_slab_ptr+0x20/0x90
Sep 16 13:10:26 node5 kernel: [29355.921400]  [<ffffffff810b764c>] ? ub_slab_charge+0xac/0x100
Sep 16 13:10:26 node5 kernel: [29355.921405]  [<ffffffff81240249>] ? ipc_lock+0x39/0x60
Sep 16 13:10:26 node5 kernel: [29355.921410]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.921414]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.921420]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.921424]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.921429]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.921434]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.921438]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.921444]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.921449]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.921454]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.921457] apache2       D ffff881ff1104c30     0 170866  25957  283 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.921462]  ffff880a99da9b38 0000000000000082 ffff8802dcc7f4a8 ffff8802dcc7f4a8
Sep 16 13:10:26 node5 kernel: [29355.921469]  ffff880a99da9b28 ffffffff8116b199 000000202eb9c025 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.921476]  ffffea00b728cb90 0000020000000030 ffff881ff11051f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.921482] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.921487]  [<ffffffff8116b199>] ? __do_fault+0x549/0x640
Sep 16 13:10:26 node5 kernel: [29355.921492]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.921498]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.921502]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.921507]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.921512]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.921517]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.921522]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.921526]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.921531]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.921536]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.921540]  [<ffffffff8104a8df>] ? __do_page_fault+0x26f/0x480
Sep 16 13:10:26 node5 kernel: [29355.921546]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.921551]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.921556]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.921560]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.921565]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.921570]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.921575]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.921579] apache2       D ffff8838ce557370     0 170867  98640 1223 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.921584]  ffff88394da11a88 0000000000000086 ffff880000034d88 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.921590]  ffff88394da10000 000000000001b720 0000000000000004 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.921597]  ffff882073660b70 000200da00000000 ffff8838ce557938 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.921604] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.921608]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.921614]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.921618]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.921623]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.921629]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.921633]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.921639]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.921644]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.921649]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.921654]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.921659]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.921664]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.921669]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.921674]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.921680]  [<ffffffff8115bd05>] ? vma_prio_tree_add+0x75/0xd0
Sep 16 13:10:26 node5 kernel: [29355.921685]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.921689]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.921694]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.921699]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.921704]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.921709]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.921713]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.921717] apache2       D ffff8838ce556880     0 170868  98640 1223 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.921722]  ffff8827ba58fa88 0000000000000086 ffff880000034d88 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.921729]  ffff8827ba58e000 000000000001b720 0000000000000000 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.921735]  ffff882073660b70 000200da00000000 ffff8838ce556e48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.921742] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.921747]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.921752]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.921757]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.921761]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.921766]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.921772]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.921777]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.921782]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.921787]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.921792]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.921797]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.921802]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.921807]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.921812]  [<ffffffff8115bd05>] ? vma_prio_tree_add+0x75/0xd0
Sep 16 13:10:26 node5 kernel: [29355.921817]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.921821]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.921826]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.921831]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.921836]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.921841]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.921845]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.921849] apache2       D ffff883e449b93b0     0 170869  98640 1223 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.921855]  ffff8839a371fa88 0000000000000082 ffff880000034d88 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.921861]  ffff8839a371e000 000000000001b720 0000000000000200 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.921868]  ffff882073660b70 000200da00000000 ffff883e449b9978 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.921874] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.921879]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.921884]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.921889]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.921893]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.921898]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.921904]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.921909]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.921914]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.921919]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.921924]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.921929]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.921934]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.921939]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.921944]  [<ffffffff8115bd05>] ? vma_prio_tree_add+0x75/0xd0
Sep 16 13:10:26 node5 kernel: [29355.921949]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.921953]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.921958]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.921963]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.921968]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.921973]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.921978]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.921981] apache2       D ffff883e449b88c0     0 170870  98640 1223 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.921986]  ffff88361bb4ba88 0000000000000086 ffff882080022c48 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.921993]  ffff88361bb4a000 000000000001b720 0000000000000000 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.922000]  ffff884073493a70 000200da00000000 ffff883e449b8e88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.922006] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.922011]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.922016]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.922021]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.922025]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.922030]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.922035]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.922040]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.922046]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.922051]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.922056]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.922061]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.922066]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.922071]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.922076]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.922081]  [<ffffffff8115bd05>] ? vma_prio_tree_add+0x75/0xd0
Sep 16 13:10:26 node5 kernel: [29355.922086]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.922090]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.922095]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.922100]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.922105]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.922110]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.922114]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.922118] apache2       D ffff880debe18c70     0 170872  50154  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.922123]  ffff880d6361fb38 0000000000000082 0000000000000000 ffff88407346fe40
Sep 16 13:10:26 node5 kernel: [29355.922130]  ffff880d6361faf8 0000000000000246 ffff88407346fe70 000280da00000000
Sep 16 13:10:26 node5 kernel: [29355.922136]  0000000000000000 0000000101badab7 ffff880debe19238 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.922143] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.922147]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.922153]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.922157]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.922162]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.922167]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.922171]  [<ffffffff811ba8c2>] ? acl_permission_check+0x22/0xc0
Sep 16 13:10:26 node5 kernel: [29355.922176]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.922182]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.922186]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.922191]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.922196]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.922200]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.922205]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.922210]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.922214]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.922219]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.922223]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.922229]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.922233] miniserv.pl   D ffff882dbb688740     0 170873 104956 1214 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.922238]  ffff883ee4181a98 0000000000000086 ffff883ee4180000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.922244]  0000000000000200 0000000000000002 ffff882dbb689028 000200da00000000
Sep 16 13:10:26 node5 kernel: [29355.922251]  0000000000000282 0000000000000002 ffff882dbb688d08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.922257] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.922267]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.922274]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.922298]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.922323]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.922348]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.922372]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.922400]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.922405]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.922434]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.922460]  [<ffffffffa02ea8b9>] dmu_buf_hold_array+0x39/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.922468]  [<ffffffffa002a131>] ? avl_find+0x1/0xb0 [zavl]
Sep 16 13:10:26 node5 kernel: [29355.922494]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.922530]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.922566]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.922571]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.922576]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.922582]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.922586] apache2       D ffff881ff1104140     0 170874  25957  283 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.922591]  ffff880694e5db38 0000000000000086 0000000000000000 000000000000000e
Sep 16 13:10:26 node5 kernel: [29355.922598]  ffff880694e5db28 ffffffff8114e385 000000202eb9c025 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.922604]  0000000000000000 0000000101baa180 ffff881ff1104708 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.922611] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.922615]  [<ffffffff8114e385>] ? release_pages+0x255/0x3f0
Sep 16 13:10:26 node5 kernel: [29355.922623]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.922627]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.922632]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.922637]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.922642]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.922647]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.922652]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.922656]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.922661]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.922666]  [<ffffffff811bff69>] ? do_filp_open+0xc9/0xc50
Sep 16 13:10:26 node5 kernel: [29355.922671]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.922676]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.922681]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.922686]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.922691]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.922696]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.922701]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.922706]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.922709] apache2       D ffff880eea09acb0     0 170875  25957  283 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.922714]  ffff88150d563b38 0000000000000082 ffff88150d563b00 ffff880ee731e9e0
Sep 16 13:10:26 node5 kernel: [29355.922721]  ffff88150d563b28 00000000011a48cf ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.922728]  ffff8820737b0800 0000000101baa17e ffff880eea09b278 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.922735] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.922739]  [<ffffffff8152efd1>] ? schedule+0x1/0x44a
Sep 16 13:10:26 node5 kernel: [29355.922745]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.922749]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.922754]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.922759]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.922763]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.922769]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.922773]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.922778]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.922783]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.922788]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.922792]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.922798]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.922803]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.922807]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.922812]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.922817]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.922822]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.922827]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.922830] apache2       D ffff880debe18180     0 170876  50154  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.922835]  ffff88085fbc3b38 0000000000000086 ffff88085fbc3b00 ffff881691423840
Sep 16 13:10:26 node5 kernel: [29355.922842]  0000000000000000 00000000011d50e1 ffff882100d10770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.922849]  ffff882100d10760 0000000101baf1e4 ffff880debe18748 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.922855] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.922859]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.922865]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.922870]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.922875]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.922879]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.922884]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.922889]  [<ffffffff811ba788>] ? get_fs_root+0x48/0x80
Sep 16 13:10:26 node5 kernel: [29355.922893]  [<ffffffff811ba76b>] ? get_fs_root+0x2b/0x80
Sep 16 13:10:26 node5 kernel: [29355.922898]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.922902]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.922907]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.922912]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.922917]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.922922]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.922927]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.922931]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.922936]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.922941]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.922946]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.922953] apache2       D ffff8808d6df9230     0 170877  50154  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.922958]  ffff88177f9c1b38 0000000000000082 0000000000000000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.922965]  0000000000000000 000000000036f3b5 ffff882100c50230 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.922972]  0000000000000000 0000000101bb11f8 ffff8808d6df97f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.922978] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.922982]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.922988]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.922993]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.922998]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.923002]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.923007]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.923012]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.923017]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.923022]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.923026]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.923031]  [<ffffffff810a70ea>] ? down_read_trylock+0x1a/0x30
Sep 16 13:10:26 node5 kernel: [29355.923036]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.923041]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.923045]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.923050]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.923054]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.923059]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.923064]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.923068] dovecot       D ffff8805da84e900     0 170878 127351 1297 0x00000004
Sep 16 13:10:26 node5 kernel: [29355.923073]  ffff8818d90ff288 0000000000000086 ffff8818d90ff250 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.923080]  0000000000000000 00000000009e005f ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.923086]  ffff8820737b0800 0000000101baa6ee ffff8805da84eec8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.923093] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.923102]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.923109]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.923132]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923158]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923183]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923207]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923235]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923264]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923290]  [<ffffffffa02e969a>] dmu_buf_hold_noread+0x3a/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923316]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923351]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923387]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923421]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923455]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923488]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923522]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923527]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.923562]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923596]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923601]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.923606]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.923611]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.923616]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.923621]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.923627]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.923631]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.923638]  [<ffffffff810923e6>] ? virtinfo_notifier_call+0x76/0xb0
Sep 16 13:10:26 node5 kernel: [29355.923672]  [<ffffffffa038733f>] ? zpl_read+0x10f/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.923677]  [<ffffffff811b67b0>] open_exec+0x50/0x120
Sep 16 13:10:26 node5 kernel: [29355.923682]  [<ffffffff8120d8f8>] load_elf_binary+0x1628/0x1a10
Sep 16 13:10:26 node5 kernel: [29355.923687]  [<ffffffff81165853>] ? follow_page+0x413/0x500
Sep 16 13:10:26 node5 kernel: [29355.923692]  [<ffffffff8116c6e0>] ? __get_user_pages+0x110/0x410
Sep 16 13:10:26 node5 kernel: [29355.923697]  [<ffffffff8120990f>] ? load_misc_binary+0xaf/0x400
Sep 16 13:10:26 node5 kernel: [29355.923702]  [<ffffffff8116ca84>] ? get_user_pages+0x54/0x60
Sep 16 13:10:26 node5 kernel: [29355.923707]  [<ffffffff811b53bc>] ? get_arg_page+0x5c/0x100
Sep 16 13:10:26 node5 kernel: [29355.923712]  [<ffffffff8129ac88>] ? strnlen_user+0x38/0x80
Sep 16 13:10:26 node5 kernel: [29355.923717]  [<ffffffff811b60f7>] search_binary_handler+0x127/0x370
Sep 16 13:10:26 node5 kernel: [29355.923722]  [<ffffffff811b6aa3>] do_execve+0x223/0x2d0
Sep 16 13:10:26 node5 kernel: [29355.923728]  [<ffffffff8100966a>] sys_execve+0x4a/0x80
Sep 16 13:10:26 node5 kernel: [29355.923732]  [<ffffffff8100b55a>] stub_execve+0x6a/0xc0
Sep 16 13:10:26 node5 kernel: [29355.923736] apache2       D ffff880466b77130     0 170879  50154  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.923741]  ffff88031c8adb38 0000000000000086 0000000000000000 ffff88207fc025c0
Sep 16 13:10:26 node5 kernel: [29355.923748]  ffff88031c8adad8 ffffffff81194089 0000000000000157 ffff88207fc033c0
Sep 16 13:10:26 node5 kernel: [29355.923754]  0000000000000000 0000000101bb12f9 ffff880466b776f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.923762] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.923765]  [<ffffffff81194089>] ? __drain_alien_cache+0x89/0xa0
Sep 16 13:10:26 node5 kernel: [29355.923772]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.923777]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.923782]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.923786]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.923791]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.923797]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.923801]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.923806]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.923811]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.923815]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.923820]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.923825]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.923829]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.923834]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.923838]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.923844]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.923849] apache2       D ffff88169a57abf0     0 170880  50154  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.923854]  ffff880a9e103b38 0000000000000082 0000000000000000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.923860]  000000009e103c28 0000000000646206 ffff882100c50230 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.923867]  0000000000000000 0000000101bb391c ffff88169a57b1b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.923874] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.923879]  [<ffffffff814b00b0>] ? tcp_cleanup_rbuf+0x90/0x140
Sep 16 13:10:26 node5 kernel: [29355.923884]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.923889]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.923894]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.923898]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.923903]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.923908]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.923913]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.923918]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.923923]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.923928]  [<ffffffff8116f236>] ? find_vma+0x46/0x80
Sep 16 13:10:26 node5 kernel: [29355.923932]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.923937]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.923941]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.923946]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.923951]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.923955]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.923960]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.923965] apache2       D ffff88018a9cec30     0 170881  50154  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.923969]  ffff88156ac45b38 0000000000000086 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.923976]  ffff88407357a6d0 ffff880c6de7d800 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.923983]  0000000000000000 0000000101bb514c ffff88018a9cf1f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.923989] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.923994]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.923999]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.924004]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.924009]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.924013]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.924018]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.924023]  [<ffffffff811ba76b>] ? get_fs_root+0x2b/0x80
Sep 16 13:10:26 node5 kernel: [29355.924027]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.924032]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.924037]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.924042]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.924047]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.924052]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.924057]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.924062]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.924067]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.924071]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.924076]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.924081]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.924085] apache2       D ffff8812f3706e70     0 170882  50154  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.924090]  ffff881055ee5b38 0000000000000086 0000000000000000 ffff880c998982f8
Sep 16 13:10:26 node5 kernel: [29355.924097]  00007f488a75a7a8 ffffffffffffff03 ffff882100c10770 ffff881055ee5b38
Sep 16 13:10:26 node5 kernel: [29355.924103]  ffff882100c10760 0000000101bb6e96 ffff8812f3707438 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.924110] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.924114]  [<ffffffff81055a18>] ? mutex_spin_on_owner+0xb8/0xc0
Sep 16 13:10:26 node5 kernel: [29355.924120]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.924124]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.924129]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.924134]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.924138]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.924143]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.924148]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.924153]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.924158]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.924162]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.924167]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.924171]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.924176]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.924181]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.924185]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.924190]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.924195] /usr/sbin/mun D ffff880eea09a1c0     0 170887 166781  736 0x00000004
Sep 16 13:10:26 node5 kernel: [29355.924200]  ffff880df88d7578 0000000000000082 0000000000000001 0000004100000000
Sep 16 13:10:26 node5 kernel: [29355.924207]  ffff882080022c48 ffff880df88d7508 ffffffff81160a49 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.924213]  ffff884073ee0a40 ffff882080011640 ffff880eea09a788 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.924220] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.924224]  [<ffffffff81160a49>] ? zone_statistics+0x99/0xc0
Sep 16 13:10:26 node5 kernel: [29355.924234]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.924240]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.924264]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924270]  [<ffffffff81160a49>] ? zone_statistics+0x99/0xc0
Sep 16 13:10:26 node5 kernel: [29355.924293]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924318]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924342]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924371]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924399]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924424]  [<ffffffffa02e969a>] dmu_buf_hold_noread+0x3a/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924450]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924486]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924491]  [<ffffffff81142773>] ? __rmqueue+0xc3/0x490
Sep 16 13:10:26 node5 kernel: [29355.924525]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924559]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924593]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924627]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924660]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924665]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.924670]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.924705]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924740]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924745]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.924749]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.924755]  [<ffffffff8116b55e>] ? handle_pte_fault+0x2ce/0x1040
Sep 16 13:10:26 node5 kernel: [29355.924760]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.924765]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.924769]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.924774]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.924779]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.924784]  [<ffffffff812855a7>] ? cic_free_func+0x87/0xa0
Sep 16 13:10:26 node5 kernel: [29355.924789]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.924794]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.924798]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.924803]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.924807]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.924812]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.924818]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.924822] apache2       S ffff881dcbd842c0     0 170888  47942  526 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.924827]  ffff880a8a50fcf8 0000000000000086 0000000000000000 00007f57e7b37250
Sep 16 13:10:26 node5 kernel: [29355.924833]  ffff880a8a50fc78 ffffffff81193a10 ffff8840734c2410 ffff88207fc07380
Sep 16 13:10:26 node5 kernel: [29355.924840]  ffff8840734c2400 0000000101baae5c ffff881dcbd84888 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.924847] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.924850]  [<ffffffff81193a10>] ? ub_slab_ptr+0x20/0x90
Sep 16 13:10:26 node5 kernel: [29355.924856]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.924861]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.924866]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.924871]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.924876]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.924881]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.924886]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.924891]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.924897]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.924900] apache2       D ffff8801ebe31430     0 170889  81474 1127 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.924905]  ffff88052e82d618 0000000000000082 ffff88052e82d5e0 0000000000000286
Sep 16 13:10:26 node5 kernel: [29355.924912]  ffff88052e82d598 000000000112aa00 ffff882100cd0770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.924919]  ffff882100cd0760 0000000101baafb1 ffff8801ebe319f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.924925] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.924935]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.924943]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.924966]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.924973]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.924997]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925021]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925044]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925070]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925096]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925131]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925137]  [<ffffffff811442e3>] ? get_page_from_freelist+0x3d3/0x8a0
Sep 16 13:10:26 node5 kernel: [29355.925171]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925204]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925237]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925271]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925304]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925309]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.925344]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925379]  [<ffffffffa0370de1>] ? zfs_lookup+0x1/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925413]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925418]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.925423]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.925428]  [<ffffffff8116b55e>] ? handle_pte_fault+0x2ce/0x1040
Sep 16 13:10:26 node5 kernel: [29355.925436]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.925441]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.925449]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.925454]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.925459]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.925465]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.925470]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.925474]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.925479]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.925485]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.925490]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.925495]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.925499] apache2       S ffff881a483b6cf0     0 170890  47942  526 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.925504]  ffff88082ec05cf8 0000000000000082 ffff8809fc5299b8 00007f57e7b37250
Sep 16 13:10:26 node5 kernel: [29355.925511]  ffff88082ec05c78 ffffffff81193a10 ffff883fe8960100 ffff88207fc07380
Sep 16 13:10:26 node5 kernel: [29355.925518]  ffffffff8100beae ffff88082ec05f68 ffff881a483b72b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.925524] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.925528]  [<ffffffff81193a10>] ? ub_slab_ptr+0x20/0x90
Sep 16 13:10:26 node5 kernel: [29355.925533]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.925538]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.925543]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.925548]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.925553]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.925558]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.925564]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.925569]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.925574]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.925579]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.925583] apache2       S ffff881a483b6200     0 170891  47942  526 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.925588]  ffff8819c04d1cf8 0000000000000086 0000000000000000 00007f57e7b37250
Sep 16 13:10:26 node5 kernel: [29355.925595]  ffff8819c04d1c78 ffffffff81193a10 ffff883fe8960100 ffff88207fc07380
Sep 16 13:10:26 node5 kernel: [29355.925601]  0000000000000000 0000000101bab1a7 ffff881a483b67c8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.925608] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.925612]  [<ffffffff81193a10>] ? ub_slab_ptr+0x20/0x90
Sep 16 13:10:26 node5 kernel: [29355.925617]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.925622]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.925627]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.925632]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.925637]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.925642]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.925647]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.925652]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.925657]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.925661] httpd.itk     D ffff881727f31470     0 170894 170196 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.925666]  ffff8809cd59d618 0000000000000082 ffffffff8100beae ffff8809cd59d618
Sep 16 13:10:26 node5 kernel: [29355.925673]  0000000000000000 ffff8809cd59d7b0 ffff880cf1079800 ffffffffa03c3f38
Sep 16 13:10:26 node5 kernel: [29355.925680]  000000000000035d ffff88012bf5ec80 ffff881727f31a38 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.925686] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.925690]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.925701]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.925708]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.925732]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925756]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925780]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925804]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925830]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925856]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925861]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.925894]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925923]  [<ffffffffa02fa8d4>] ? dnode_rele+0x54/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.925928]  [<ffffffff81532527>] ? _spin_unlock_irqrestore+0x17/0x20
Sep 16 13:10:26 node5 kernel: [29355.925933]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.925967]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.926000]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.926033]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.926067]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.926099]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.926105]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.926139]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.926173]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.926178]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.926183]  [<ffffffff811ba8db>] ? acl_permission_check+0x3b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.926188]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.926193]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.926198]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.926203]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.926208]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.926212]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.926218]  [<ffffffff8117014b>] ? vma_link+0x9b/0xf0
Sep 16 13:10:26 node5 kernel: [29355.926222]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.926227]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.926232]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.926236]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.926241]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.926246]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.926251] httpd.itk     S ffff881727f30980     0 170895  73046 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.926256]  ffff881a46ad9e88 0000000000000086 00007ff4cbb58808 ffff8808d6d1b400
Sep 16 13:10:26 node5 kernel: [29355.926263]  ffff880bc8e493a0 0000000000000007 ffff881a46ad9f18 ffffffff8104a85c
Sep 16 13:10:26 node5 kernel: [29355.926269]  0000000000000286 ffff881a46ad9e50 ffff881727f30f48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.926276] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.926280]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.926285]  [<ffffffff8107a917>] do_wait+0x1f7/0x260
Sep 16 13:10:26 node5 kernel: [29355.926289]  [<ffffffff8107aa08>] sys_wait4+0x88/0xd0
Sep 16 13:10:26 node5 kernel: [29355.926295]  [<ffffffff81078c70>] ? child_wait_callback+0x0/0x90
Sep 16 13:10:26 node5 kernel: [29355.926299]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.926305]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.926309] flush-zfs-14  S ffff8816c3050640     0 170896      2    0 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.926314]  ffff881a48335da0 0000000000000046 0000000000000000 ffff881a48335d78
Sep 16 13:10:26 node5 kernel: [29355.926320]  0000000101bb7d48 0000000000000400 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.926327]  0000000000000000 0000000000000024 ffff8816c3050c08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.926333] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.926338]  [<ffffffff81530902>] schedule_timeout+0x192/0x2e0
Sep 16 13:10:26 node5 kernel: [29355.926344]  [<ffffffff81088830>] ? process_timeout+0x0/0x10
Sep 16 13:10:26 node5 kernel: [29355.926349]  [<ffffffff811dcf3e>] bdi_writeback_task+0x14e/0x1e0
Sep 16 13:10:26 node5 kernel: [29355.926353]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.926358]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.926362]  [<ffffffff81162285>] bdi_start_fn+0x95/0x110
Sep 16 13:10:26 node5 kernel: [29355.926366]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.926372]  [<ffffffff810a15e6>] kthread+0x96/0xa0
Sep 16 13:10:26 node5 kernel: [29355.926377]  [<ffffffff810a1550>] ? kthread+0x0/0xa0
Sep 16 13:10:26 node5 kernel: [29355.926381]  [<ffffffff8100c34a>] child_rip+0xa/0x20
Sep 16 13:10:26 node5 kernel: [29355.926386]  [<ffffffff810a1550>] ? kthread+0x0/0xa0
Sep 16 13:10:26 node5 kernel: [29355.926390]  [<ffffffff8100c340>] ? child_rip+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.926394] httpd.itk     D ffff8814d7e874b0     0 170898 170254 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.926399]  ffff8815af535b38 0000000000000086 ffff8815af534000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.926406]  00000000af535ac8 0000000000000002 ffff8820735fe9f0 000280da00000000
Sep 16 13:10:26 node5 kernel: [29355.926412]  0000000000000286 0000000000000030 ffff8814d7e87a78 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.926419] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.926424]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.926429]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.926434]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.926439]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.926443]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.926449]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.926453]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.926458]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.926463]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.926467]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.926472]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.926477]  [<ffffffff8117014b>] ? vma_link+0x9b/0xf0
Sep 16 13:10:26 node5 kernel: [29355.926482]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.926486]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.926491]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.926495]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.926500]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.926505]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.926509] httpd.itk     S ffff8805da84f3f0     0 170899  73046 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.926514]  ffff8815698dfe88 0000000000000086 00007ff4cbb58808 ffff8801595a42c0
Sep 16 13:10:26 node5 kernel: [29355.926520]  ffff881a48aaa318 0000000000000007 ffffffff8100beae ffff8815698dfe88
Sep 16 13:10:26 node5 kernel: [29355.926527]  0000000000000246 0000000000000000 ffff8805da84f9b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.926534] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.926538]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.926543]  [<ffffffff8153238d>] ? _read_lock+0xd/0x20
Sep 16 13:10:26 node5 kernel: [29355.926547]  [<ffffffff8107a917>] do_wait+0x1f7/0x260
Sep 16 13:10:26 node5 kernel: [29355.926551]  [<ffffffff8107aa08>] sys_wait4+0x88/0xd0
Sep 16 13:10:26 node5 kernel: [29355.926556]  [<ffffffff81078c70>] ? child_wait_callback+0x0/0x90
Sep 16 13:10:26 node5 kernel: [29355.926561]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.926566]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.926570] httpd.itk     S ffff881055ed51f0     0 170900  73046 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.926575]  ffff880fca5cfda8 0000000000000086 ffff880fca5cfd70 ffffffff810a6ad3
Sep 16 13:10:26 node5 kernel: [29355.926581]  0000000000000013 0000000000d0e0fc ffff882100d50770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.926588]  ffff882100d50760 0000000101bb7c4c ffff881055ed57b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.926594] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.926599]  [<ffffffff810a6ad3>] ? __hrtimer_start_range_ns+0x1a3/0x460
Sep 16 13:10:26 node5 kernel: [29355.926605]  [<ffffffff81531578>] schedule_hrtimeout_range+0xc8/0x160
Sep 16 13:10:26 node5 kernel: [29355.926610]  [<ffffffff810a5dc0>] ? hrtimer_wakeup+0x0/0x30
Sep 16 13:10:26 node5 kernel: [29355.926614]  [<ffffffff810a6dc4>] ? hrtimer_start_range_ns+0x14/0x20
Sep 16 13:10:26 node5 kernel: [29355.926620]  [<ffffffff811f91ac>] ep_poll+0x2bc/0x350
Sep 16 13:10:26 node5 kernel: [29355.926625]  [<ffffffff81060020>] ? default_wake_function+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.926630]  [<ffffffff811f9305>] sys_epoll_wait+0xc5/0xe0
Sep 16 13:10:26 node5 kernel: [29355.926635]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.926639] apache2       D ffff882f1b33a640     0 170901 1002180 2029 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.926644]  ffff8833bc38ba88 0000000000000086 ffff880000034d88 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.926651]  ffff8833bc38a000 000000000001b720 00000000001c0bbe 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.926658]  ffff882073660b70 000200da00000000 ffff882f1b33ac08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.926664] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.926669]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.926675]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.926680]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.926684]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.926689]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.926695]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.926700]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.926705]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.926711]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.926716]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.926721]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.926726]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.926731]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.926736]  [<ffffffff8115bd05>] ? vma_prio_tree_add+0x75/0xd0
Sep 16 13:10:26 node5 kernel: [29355.926741]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.926746]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.926750]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.926756]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.926761]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.926766]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.926770]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.926774] php           S ffff88118612a440     0 170905 153634 1592 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.926779]  ffff8805611e5c98 0000000000000086 ffffea0062976680 ffffea0000000000
Sep 16 13:10:26 node5 kernel: [29355.926786]  ffff8805611e5c18 ffffffff811350b7 ffff881df3bfd570 ffff881df3bfd570
Sep 16 13:10:26 node5 kernel: [29355.926793]  ffff8805611e5ca8 ffffffff8116b199 ffff88118612aa08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.926799] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.926803]  [<ffffffff811350b7>] ? unlock_page+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.926808]  [<ffffffff8116b199>] ? __do_fault+0x549/0x640
Sep 16 13:10:26 node5 kernel: [29355.926813]  [<ffffffff810a1f2e>] ? prepare_to_wait+0x4e/0x80
Sep 16 13:10:26 node5 kernel: [29355.926819]  [<ffffffff811b7fbb>] pipe_wait+0x5b/0x80
Sep 16 13:10:26 node5 kernel: [29355.926824]  [<ffffffff810a1c00>] ? autoremove_wake_function+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.926829]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.926834]  [<ffffffff811b8a56>] pipe_read+0x3e6/0x4e0
Sep 16 13:10:26 node5 kernel: [29355.926839]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.926844]  [<ffffffff811ac78a>] do_sync_read+0xfa/0x140
Sep 16 13:10:26 node5 kernel: [29355.926849]  [<ffffffff810a1c00>] ? autoremove_wake_function+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.926855]  [<ffffffff81074745>] ? do_fork_pid+0x145/0x490
Sep 16 13:10:26 node5 kernel: [29355.926860]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.926865]  [<ffffffff811ae55e>] ? alloc_file+0x2e/0xe0
Sep 16 13:10:26 node5 kernel: [29355.926870]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.926875]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.926879]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.926885]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.926889] php           S ffff8808d6df8740     0 170906 153634 1592 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.926894]  ffff88052c2a5c98 0000000000000082 ffffea0062976680 ffffea0000000000
Sep 16 13:10:26 node5 kernel: [29355.926901]  ffff88052c2a5c18 ffffffff811350b7 ffff8804e5fb8b70 ffff8804e5fb8b70
Sep 16 13:10:26 node5 kernel: [29355.926908]  ffff88052c2a5ca8 ffffffff8116b199 ffff8808d6df8d08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.926915] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.926919]  [<ffffffff811350b7>] ? unlock_page+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.926924]  [<ffffffff8116b199>] ? __do_fault+0x549/0x640
Sep 16 13:10:26 node5 kernel: [29355.926929]  [<ffffffff810a1f2e>] ? prepare_to_wait+0x4e/0x80
Sep 16 13:10:26 node5 kernel: [29355.926934]  [<ffffffff811b7fbb>] pipe_wait+0x5b/0x80
Sep 16 13:10:26 node5 kernel: [29355.926939]  [<ffffffff810a1c00>] ? autoremove_wake_function+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.926944]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.926949]  [<ffffffff811b8a56>] pipe_read+0x3e6/0x4e0
Sep 16 13:10:26 node5 kernel: [29355.926955]  [<ffffffff811ac78a>] do_sync_read+0xfa/0x140
Sep 16 13:10:26 node5 kernel: [29355.926960]  [<ffffffff810a1c00>] ? autoremove_wake_function+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.926965]  [<ffffffff81074745>] ? do_fork_pid+0x145/0x490
Sep 16 13:10:26 node5 kernel: [29355.926970]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.926975]  [<ffffffff811ae55e>] ? alloc_file+0x2e/0xe0
Sep 16 13:10:26 node5 kernel: [29355.926980]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.926985]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.926990]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.926995]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.927001] sh            D ffff88104a3fa340     0 170907 170905 1592 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.927005]  ffff8817d9979378 0000000000000082 ffff8817d9979308 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.927012]  00000000000000a2 ffff88088a4bade0 ffff8817d9979308 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.927019]  0000000000004030 ffff884071850000 ffff88104a3fa908 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.927025] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.927060]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927070]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.927077]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.927100]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927125]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927150]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927174]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927201]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927228]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927234]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.927269]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927304]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927309]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.927314]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.927347]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927380]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927385]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.927390]  [<ffffffff8114cf8c>] ? max_sane_readahead+0x2c/0xd0
Sep 16 13:10:26 node5 kernel: [29355.927395]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.927400]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.927405]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.927410]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.927415]  [<ffffffff8118c3ea>] ? alloc_pages_current+0xaa/0x110
Sep 16 13:10:26 node5 kernel: [29355.927420]  [<ffffffff8104eb17>] ? pte_alloc_one+0x37/0x50
Sep 16 13:10:26 node5 kernel: [29355.927425]  [<ffffffff811679a2>] ? __pte_alloc+0xd2/0x350
Sep 16 13:10:26 node5 kernel: [29355.927430]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.927435]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.927439]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.927445]  [<ffffffff8117003c>] ? __vma_link_file+0x4c/0x80
Sep 16 13:10:26 node5 kernel: [29355.927450]  [<ffffffff8117014b>] ? vma_link+0x9b/0xf0
Sep 16 13:10:26 node5 kernel: [29355.927454]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.927459]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.927464]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.927469]  [<ffffffff8129adcf>] ? __clear_user+0x3f/0x70
Sep 16 13:10:26 node5 kernel: [29355.927474]  [<ffffffff8129adb1>] ? __clear_user+0x21/0x70
Sep 16 13:10:26 node5 kernel: [29355.927479]  [<ffffffff8129ae38>] clear_user+0x38/0x40
Sep 16 13:10:26 node5 kernel: [29355.927483]  [<ffffffff8120be9d>] padzero+0x2d/0x40
Sep 16 13:10:26 node5 kernel: [29355.927488]  [<ffffffff8120caa7>] load_elf_binary+0x7d7/0x1a10
Sep 16 13:10:26 node5 kernel: [29355.927493]  [<ffffffff81165853>] ? follow_page+0x413/0x500
Sep 16 13:10:26 node5 kernel: [29355.927498]  [<ffffffff8116c6e0>] ? __get_user_pages+0x110/0x410
Sep 16 13:10:26 node5 kernel: [29355.927503]  [<ffffffff8120990f>] ? load_misc_binary+0xaf/0x400
Sep 16 13:10:26 node5 kernel: [29355.927508]  [<ffffffff8129ac88>] ? strnlen_user+0x38/0x80
Sep 16 13:10:26 node5 kernel: [29355.927513]  [<ffffffff811b60f7>] search_binary_handler+0x127/0x370
Sep 16 13:10:26 node5 kernel: [29355.927518]  [<ffffffff811b6aa3>] do_execve+0x223/0x2d0
Sep 16 13:10:26 node5 kernel: [29355.927524]  [<ffffffff8100966a>] sys_execve+0x4a/0x80
Sep 16 13:10:26 node5 kernel: [29355.927528]  [<ffffffff8100b55a>] stub_execve+0x6a/0xc0
Sep 16 13:10:26 node5 kernel: [29355.927532] sh            D ffff880528e0cff0     0 170908 170906 1592 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.927537]  ffff880899975378 0000000000000086 0000000000000018 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.927544]  ffff8808998ccab8 ffff88088a4bb210 ffff880899975308 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.927551]  0000000000004030 ffff884071850000 ffff880528e0d5b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.927558] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.927590]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927603]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.927609]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.927632]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927657]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927682]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927706]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927732]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927760]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927764]  [<ffffffff81292ac3>] ? radix_tree_preload+0x23/0xa0
Sep 16 13:10:26 node5 kernel: [29355.927769]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.927804]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927838]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927844]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.927849]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.927881]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927913]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.927918]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.927923]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.927928]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.927933]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.927938]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.927943]  [<ffffffff8118c3ea>] ? alloc_pages_current+0xaa/0x110
Sep 16 13:10:26 node5 kernel: [29355.927948]  [<ffffffff8104eb17>] ? pte_alloc_one+0x37/0x50
Sep 16 13:10:26 node5 kernel: [29355.927953]  [<ffffffff811679a2>] ? __pte_alloc+0xd2/0x350
Sep 16 13:10:26 node5 kernel: [29355.927958]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.927963]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.927968]  [<ffffffff8117003c>] ? __vma_link_file+0x4c/0x80
Sep 16 13:10:26 node5 kernel: [29355.927973]  [<ffffffff8117014b>] ? vma_link+0x9b/0xf0
Sep 16 13:10:26 node5 kernel: [29355.927978]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.927983]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.927987]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.927993]  [<ffffffff8129adcf>] ? __clear_user+0x3f/0x70
Sep 16 13:10:26 node5 kernel: [29355.927998]  [<ffffffff8129adb1>] ? __clear_user+0x21/0x70
Sep 16 13:10:26 node5 kernel: [29355.928002]  [<ffffffff8129ae38>] clear_user+0x38/0x40
Sep 16 13:10:26 node5 kernel: [29355.928007]  [<ffffffff8120be9d>] padzero+0x2d/0x40
Sep 16 13:10:26 node5 kernel: [29355.928011]  [<ffffffff8120caa7>] load_elf_binary+0x7d7/0x1a10
Sep 16 13:10:26 node5 kernel: [29355.928016]  [<ffffffff81165853>] ? follow_page+0x413/0x500
Sep 16 13:10:26 node5 kernel: [29355.928021]  [<ffffffff8116c6e0>] ? __get_user_pages+0x110/0x410
Sep 16 13:10:26 node5 kernel: [29355.928026]  [<ffffffff8120990f>] ? load_misc_binary+0xaf/0x400
Sep 16 13:10:26 node5 kernel: [29355.928031]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.928036]  [<ffffffff8114e6e4>] ? put_page+0x14/0x40
Sep 16 13:10:26 node5 kernel: [29355.928041]  [<ffffffff811b60f7>] search_binary_handler+0x127/0x370
Sep 16 13:10:26 node5 kernel: [29355.928046]  [<ffffffff811b6aa3>] do_execve+0x223/0x2d0
Sep 16 13:10:26 node5 kernel: [29355.928052]  [<ffffffff8100966a>] sys_execve+0x4a/0x80
Sep 16 13:10:26 node5 kernel: [29355.928056]  [<ffffffff8100b55a>] stub_execve+0x6a/0xc0
Sep 16 13:10:26 node5 kernel: [29355.928060] nrpe          D ffff88203508ed30     0 170910 149266 2223 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.928065]  ffff882035015638 0000000000000086 0000000000000202 ffff8820350155a8
Sep 16 13:10:26 node5 kernel: [29355.928072]  ffff882035015668 ffffffffa02f9942 0000000000000041 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.928078]  ffff882035015608 ffffffff00000000 ffff88203508f2f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.928085] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.928113]  [<ffffffffa02f9942>] ? dmu_zfetch+0x4b2/0xe30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928122]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.928131]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.928154]  [<ffffffffa02d75fa>] arc_buf_add_ref+0x9a/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928178]  [<ffffffffa02e1946>] __dbuf_hold_impl+0x66/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928202]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928226]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928251]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928277]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928312]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928346]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928380]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928413]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928446]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928479]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928513]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928547]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928552]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.928557]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.928563]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.928568]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.928572]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.928578]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.928583]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.928588]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.928593]  [<ffffffff811cdaaf>] ? alloc_fd+0x1f/0x160
Sep 16 13:10:26 node5 kernel: [29355.928598]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.928602]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.928607]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.928612]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.928616] apache2       S ffff88203508e240     0 170912  72054 1088 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.928621]  ffff88054db99cf8 0000000000000082 ffff881ba7d7a908 00007fe7ba921910
Sep 16 13:10:26 node5 kernel: [29355.928628]  ffff88054db99c78 ffffffff81193a10 ffff883efd3e3400 ffff88207fc07380
Sep 16 13:10:26 node5 kernel: [29355.928635]  ffff88054db99ca8 ffffffff810b764c ffff88203508e808 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.928642] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.928645]  [<ffffffff81193a10>] ? ub_slab_ptr+0x20/0x90
Sep 16 13:10:26 node5 kernel: [29355.928651]  [<ffffffff810b764c>] ? ub_slab_charge+0xac/0x100
Sep 16 13:10:26 node5 kernel: [29355.928656]  [<ffffffff81240249>] ? ipc_lock+0x39/0x60
Sep 16 13:10:26 node5 kernel: [29355.928661]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.928666]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.928671]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.928675]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.928681]  [<ffffffff811ae2f0>] ? fget+0x30/0x80
Sep 16 13:10:26 node5 kernel: [29355.928685]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.928691]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.928696]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.928699] apache2       D ffff880399286db0     0 170914  99398 3205 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.928704]  ffff880493397a88 0000000000000082 ffff880493397a50 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.928711]  ffff880493396000 00000000007905ba ffff882100c90770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.928718]  ffff882100c90760 0000000101bad3a9 ffff880399287378 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.928725] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.928730]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.928735]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.928740]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.928745]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.928750]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.928755]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.928761]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.928766]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.928771]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.928776]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.928781]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.928786]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.928791]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.928797]  [<ffffffff8115bd05>] ? vma_prio_tree_add+0x75/0xd0
Sep 16 13:10:26 node5 kernel: [29355.928802]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.928806]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.928811]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.928817]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.928822]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.928827]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.928831]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.928835] apache2       D ffff8811b344e280     0 170915  69615 1177 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.928840]  ffff88038a10f638 0000000000000086 ffff8801bb89aec0 ffff88206dd892d0
Sep 16 13:10:26 node5 kernel: [29355.928847]  ffff88038a10f668 ffffffffa02f9ae6 000000010194c193 ffffffff00000001
Sep 16 13:10:26 node5 kernel: [29355.928853]  ffff88206dd892d8 ffff88206dd892d0 ffff8811b344e848 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.928860] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.928888]  [<ffffffffa02f9ae6>] ? dmu_zfetch+0x656/0xe30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928898]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.928906]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.928929]  [<ffffffffa02d75fa>] arc_buf_add_ref+0x9a/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928953]  [<ffffffffa02e1946>] __dbuf_hold_impl+0x66/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.928978]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.929002]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.929028]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.929053]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.929088]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.929121]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.929154]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.929187]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.929220]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.929253]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.929258]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.929292]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.929326]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.929331]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.929335]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.929341]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.929346]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.929350]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.929356]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.929360]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.929366]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.929371]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.929376]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.929381]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.929386]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.929391]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.929396]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.929401] apache2       D ffff8803992862c0     0 170916  99398 3205 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.929406]  ffff8803f9f8fa88 0000000000000086 ffff8803f9f8fa50 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.929413]  ffff8803f9f8e000 00000000005f3a7d ffff8840734c2410 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.929419]  ffff8840734c2400 0000000101bad797 ffff880399286888 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.929426] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.929431]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.929437]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.929441]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.929446]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.929451]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.929457]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.929462]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.929467]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.929472]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.929477]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.929483]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.929487]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.929492]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.929498]  [<ffffffff8115bd05>] ? vma_prio_tree_add+0x75/0xd0
Sep 16 13:10:26 node5 kernel: [29355.929503]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.929508]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.929513]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.929518]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.929523]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.929557]  [<ffffffffa0386761>] ? zpl_llseek+0x1/0x10 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.929563]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.929567]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.929571] apache2       D ffff8815bdb58df0     0 170917  99398 3205 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.929576]  ffff881448d27a88 0000000000000082 ffff881448d27a50 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.929583]  ffff881448d26000 0000000000586ed6 ffff882100c10770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.929589]  ffff882100c10760 0000000101bad799 ffff8815bdb593b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.929596] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.929601]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.929607]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.929611]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.929616]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.929621]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.929626]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.929632]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.929637]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.929642]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.929647]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.929652]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.929657]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.929662]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.929667]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.929671]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.929677]  [<ffffffff81170157>] ? vma_link+0xa7/0xf0
Sep 16 13:10:26 node5 kernel: [29355.929682]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.929687]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.929691]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.929696]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.929702]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.929707]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.929711]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.929715] /usr/sbin/mun D ffff881b214ee640     0 170918 166781  736 0x00000004
Sep 16 13:10:26 node5 kernel: [29355.929720]  ffff880a8ec27b38 0000000000000082 000000000000000e 000000000000000e
Sep 16 13:10:26 node5 kernel: [29355.929727]  ffff880a8ec27b28 ffffffff8114e385 ffffffff8100beae ffff880a8ec27b98
Sep 16 13:10:26 node5 kernel: [29355.929734]  0000000000000200 0000000000000000 ffff881b214eec08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.929740] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.929744]  [<ffffffff8114e385>] ? release_pages+0x255/0x3f0
Sep 16 13:10:26 node5 kernel: [29355.929749]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.929755]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.929759]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.929764]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.929769]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.929774]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.929779]  [<ffffffff8116b55e>] ? handle_pte_fault+0x2ce/0x1040
Sep 16 13:10:26 node5 kernel: [29355.929784]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.929789]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.929794]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.929798]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.929803]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.929808]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.929812]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.929817]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.929822]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.929826]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.929831]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.929835]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.929841]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.929845] apache2       D ffff88199fd128c0     0 170921  79040 1187 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.929850]  ffff8808d6d11b38 0000000000000086 ffff8808d6d10000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.929857]  00000000d6d11b28 0000000000000002 ffff884073493a70 000280da00000000
Sep 16 13:10:26 node5 kernel: [29355.929863]  0000000000000282 0000000000000030 ffff88199fd12e88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.929870] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.929875]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.929880]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.929885]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.929890]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.929895]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.929900]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.929904]  [<ffffffff811ba76b>] ? get_fs_root+0x2b/0x80
Sep 16 13:10:26 node5 kernel: [29355.929909]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.929913]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.929918]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.929923]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.929929]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.929934]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.929938]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.929943]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.929948]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.929953]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.929958]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.929962] nrpe          D ffff882ec42ec200     0 170923  30044  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.929967]  ffff8831f7205ab8 0000000000000082 ffff8831f7205a48 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.929973]  0000000000000282 ffff882a44ba2190 ffff8831f7205a48 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.929980]  0000000000004030 ffff884071850000 ffff882ec42ec7c8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.929987] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.930021]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930030]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.930037]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.930060]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930085]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930110]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930134]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930160]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930190]  [<ffffffffa02ea8e5>] dmu_buf_hold_array+0x65/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930216]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930251]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930256]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.930290]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930295]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.930300]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.930306]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.930311] apache2       D ffff8801a47ea6c0     0 170931  79040 1187 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.930316]  ffff880396713b38 0000000000000086 ffff880396713b00 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.930323]  0000000096713b28 00000000011aa9ff ffff88012be50770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.930329]  ffff88012be50760 0000000101bae2cb ffff8801a47eac88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.930336] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.930340]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.930346]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.930351]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.930356]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.930361]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.930366]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.930371]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.930375]  [<ffffffff811ba76b>] ? get_fs_root+0x2b/0x80
Sep 16 13:10:26 node5 kernel: [29355.930380]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.930385]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.930390]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.930394]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.930400]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.930405]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.930410]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.930415]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.930420]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.930424]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.930429]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.930433] apache2       D ffff8808ca2c2e70     0 170932  79040 1187 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.930438]  ffff880712249b38 0000000000000086 ffff880712249b00 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.930445]  0000000012249b28 0000000000d6d602 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.930452]  ffff8820737b0800 0000000101bae2c1 ffff8808ca2c3438 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.930459] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.930464]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.930470]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.930474]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.930479]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.930484]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.930489]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.930493]  [<ffffffff811ba76b>] ? get_fs_root+0x2b/0x80
Sep 16 13:10:26 node5 kernel: [29355.930498]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.930502]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.930507]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.930512]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.930517]  [<ffffffff8116f226>] ? find_vma+0x36/0x80
Sep 16 13:10:26 node5 kernel: [29355.930522]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.930527]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.930532]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.930537]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.930541]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.930546]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.930551]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.930555] apache2       D ffff8801353caeb0     0 170933  36490 2842 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.930560]  ffff88033c873578 0000000000000082 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.930566]  0000001000000000 0000000000060006 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.930573]  dead000000200200 ffff883938ef9a58 ffff8801353cb478 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.930579] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.930589]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.930595]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.930619]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930644]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930668]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930692]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930721]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930749]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930775]  [<ffffffffa02e969a>] dmu_buf_hold_noread+0x3a/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930801]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930806]  [<ffffffff811350b7>] ? unlock_page+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.930840]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930845]  [<ffffffff8114e8f1>] ? lru_cache_add_lru+0x21/0x40
Sep 16 13:10:26 node5 kernel: [29355.930850]  [<ffffffff8117701d>] ? page_add_new_anon_rmap+0x9d/0xf0
Sep 16 13:10:26 node5 kernel: [29355.930854]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.930889]  [<ffffffffa034bb21>] ? zap_lockdir+0x1/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930923]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930928]  [<ffffffff8114e8f1>] ? lru_cache_add_lru+0x21/0x40
Sep 16 13:10:26 node5 kernel: [29355.930961]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.930995]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931028]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931061]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931066]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.931100]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931134]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931139]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.931144]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.931149]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.931154]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.931159]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.931164]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.931169]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.931174]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.931180]  [<ffffffff81226d70>] ? proc_sys_delete+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.931186]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.931190]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.931195]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.931200]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.931205]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.931210]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.931215] apache2       D ffff8801353ca3c0     0 170935  36490 2842 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.931220]  ffff88033c97fb38 0000000000000082 ffff88033c97fb00 ffff88047ef4bc78
Sep 16 13:10:26 node5 kernel: [29355.931227]  ffff88033c97fb28 0000000000e872f5 ffff88012bf50770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.931233]  ffff88012bf50760 0000000101bae74b ffff8801353ca988 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.931240] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.931244]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.931250]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.931255]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.931260]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.931265]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.931270]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.931275]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.931280]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.931285]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.931289]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.931294]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.931299]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.931304]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.931309]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.931314]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.931319]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.931323]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.931328]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.931333]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.931337] apache2       D ffff880cd34e2ef0     0 170936  36490 2842 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.931342]  ffff8805a5723b38 0000000000000086 ffff8805a5723b00 ffff8806cceaf3a0
Sep 16 13:10:26 node5 kernel: [29355.931348]  ffff8805a5723b28 0000000000bb85b1 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.931355]  ffff8820737b0800 0000000101bae75a ffff880cd34e34b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.931362] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.931367]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.931371]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.931376]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.931381]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.931385]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.931391]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.931395]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.931400]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.931405]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.931410]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.931414]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.931420]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.931425]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.931430]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.931434]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.931439]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.931444]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.931449]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.931453] nrpe          D ffff88322e3bc680     0 170938 149266 2223 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.931458]  ffff8837680d5b38 0000000000000086 ffff8837680d4000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.931464]  0000000000000200 0000000000000002 ffff882073660b70 000280da00000000
Sep 16 13:10:26 node5 kernel: [29355.931471]  0000000000000282 ffff8837680d5ae8 ffff88322e3bcc48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.931478] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.931482]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.931487]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.931492]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.931497]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.931502]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.931507]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.931512]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.931516]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.931521]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.931526]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.931531]  [<ffffffff8116f226>] ? find_vma+0x36/0x80
Sep 16 13:10:26 node5 kernel: [29355.931536]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.931541]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.931546]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.931550]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.931555]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.931560]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.931564] apache2       D ffff8808ca2c2380     0 170939  22636  309 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.931569]  ffff880f2b971688 0000000000000082 dead000000200200 ffff8803d78d6e78
Sep 16 13:10:26 node5 kernel: [29355.931575]  0000002b324577da ffff88195fcbf200 ffff880f2b9715f8 ffff880f2b9715f8
Sep 16 13:10:26 node5 kernel: [29355.931582]  dead000000100100 dead000000200200 ffff8808ca2c2948 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.931589] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.931599]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.931605]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.931629]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931654]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931658]  [<ffffffff81160a49>] ? zone_statistics+0x99/0xc0
Sep 16 13:10:26 node5 kernel: [29355.931683]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931707]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931735]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931763]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931789]  [<ffffffffa02eb213>] dmu_read+0x33/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931795]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.931830]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931865]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931870]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.931875]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.931909]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931942]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.931947]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.931952]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.931957]  [<ffffffff8114d635>] ondemand_readahead+0x115/0x240
Sep 16 13:10:26 node5 kernel: [29355.931961]  [<ffffffff8114d8d6>] page_cache_sync_readahead+0x86/0xa0
Sep 16 13:10:26 node5 kernel: [29355.931967]  [<ffffffff811375fb>] filemap_fault+0x5bb/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.931972]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.931977]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.931982]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.931988]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.931993]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.931998]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.932003]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.932008]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.932014]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.932019]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.932023]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.932027] httpd.itk     D ffff8817fdbe0cf0     0 170943 170460 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.932032]  ffff881a48333b38 0000000000000082 000000000000000e 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.932039]  0000000000000002 ffff88000003dea0 ffff88207fc74840 00000000000412d0
Sep 16 13:10:26 node5 kernel: [29355.932046]  ffff881a48333c08 0000000000000213 ffff8817fdbe12b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.932052] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.932057]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.932062]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.932067]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.932072]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.932077]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.932081]  [<ffffffff81194b53>] ? cache_grow+0x1f3/0x320
Sep 16 13:10:26 node5 kernel: [29355.932086]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.932091]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.932095]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.932100]  [<ffffffff8116c2d1>] ? handle_mm_fault+0x1/0x300
Sep 16 13:10:26 node5 kernel: [29355.932105]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.932110]  [<ffffffff8117014b>] ? vma_link+0x9b/0xf0
Sep 16 13:10:26 node5 kernel: [29355.932115]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.932119]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.932124]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.932128]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.932133]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.932139]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.932142] master        D ffff883ecf1a23c0     0 170944 279918 2165 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.932148]  ffff883720ec5588 0000000000000082 ffff883720ec5550 ffff881881ad02d0
Sep 16 13:10:26 node5 kernel: [29355.932154]  ffff883720ec55b8 0000000000e973e4 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.932161]  ffff8820737b0800 0000000101baf54b ffff883ecf1a2988 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.932168] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.932177]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.932202]  [<ffffffffa02debe7>] ? dbuf_hash+0xc7/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932210]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.932233]  [<ffffffffa02d75fa>] arc_buf_add_ref+0x9a/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932257]  [<ffffffffa02e1946>] __dbuf_hold_impl+0x66/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932281]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932305]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932331]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932357]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932391]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932397]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.932431]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932464]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932497]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932530]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932563]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932568]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.932602]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932607]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.932641]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932674]  [<ffffffffa0387a41>] ? zpl_lookup+0x1/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.932679]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.932684]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.932689]  [<ffffffff81134dae>] ? find_get_page+0x1e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.932694]  [<ffffffff811350b7>] ? unlock_page+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.932699]  [<ffffffff8116b199>] ? __do_fault+0x549/0x640
Sep 16 13:10:26 node5 kernel: [29355.932703]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.932708]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.932713]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.932718]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.932722]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.932728]  [<ffffffff810a7148>] ? up_read+0x18/0x30
Sep 16 13:10:26 node5 kernel: [29355.932732]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.932737]  [<ffffffff811b67b0>] open_exec+0x50/0x120
Sep 16 13:10:26 node5 kernel: [29355.932742]  [<ffffffff811b694f>] do_execve+0xcf/0x2d0
Sep 16 13:10:26 node5 kernel: [29355.932748]  [<ffffffff8100966a>] sys_execve+0x4a/0x80
Sep 16 13:10:26 node5 kernel: [29355.932752]  [<ffffffff8100b55a>] stub_execve+0x6a/0xc0
Sep 16 13:10:26 node5 kernel: [29355.932756] apache2       D ffff881727d4cf30     0 170945  79040 1187 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.932761]  ffff88033c80bb38 0000000000000086 ffff88033c80bb00 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.932768]  000000003c80bb28 0000000002bd7200 ffff882100c10770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.932774]  ffff882100c10760 0000000101bafde3 ffff881727d4d4f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.932781] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.932786]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.932792]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.932796]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.932801]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.932807]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.932811]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.932816]  [<ffffffff811ba76b>] ? get_fs_root+0x2b/0x80
Sep 16 13:10:26 node5 kernel: [29355.932821]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.932825]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.932830]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.932835]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.932840]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.932846]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.932850]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.932855]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.932860]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.932865]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.932870]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.932873] apache2       D ffff881bb4db2f70     0 170946  25957  283 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.932879]  ffff880f2b875b38 0000000000000082 000000000000000e 000000000000000e
Sep 16 13:10:26 node5 kernel: [29355.932885]  ffff880f2b875b28 ffffffff8114e385 000000202eb9c025 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.932892]  ffffffff8100beae ffff880f2b875b48 ffff881bb4db3538 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.932899] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.932902]  [<ffffffff8114e385>] ? release_pages+0x255/0x3f0
Sep 16 13:10:26 node5 kernel: [29355.932907]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.932913]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.932918]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.932923]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.932927]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.932932]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.932937]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.932942]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.932947]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.932952]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.932957]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.932961]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.932967]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.932972]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.932977]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.932981]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.932986]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.932991]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.932996]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.933000] sshd          D ffff881727d4c440     0 170947 146662 2028 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.933005]  ffff881ab4fed288 0000000000000082 0000000000000000 ffff881ab4fed1d8
Sep 16 13:10:26 node5 kernel: [29355.933011]  dead000000100100 dead000000200200 ffff8810ae1e76f8 ffffffff811c8a60
Sep 16 13:10:26 node5 kernel: [29355.933018]  0000000000000000 0000000101bafe12 ffff881727d4ca08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.933025] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.933029]  [<ffffffff811c8a60>] ? d_callback+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.933039]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.933045]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.933069]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933094]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933119]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933143]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933171]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933199]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933225]  [<ffffffffa02e969a>] dmu_buf_hold_noread+0x3a/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933251]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933285]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933290]  [<ffffffff811c8a60>] ? d_callback+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.933325]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933330]  [<ffffffff811c8a60>] ? d_callback+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.933363]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933396]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933429]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933461]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933467]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.933501]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933535]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933540]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.933545]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.933550]  [<ffffffff811ba76b>] ? get_fs_root+0x2b/0x80
Sep 16 13:10:26 node5 kernel: [29355.933554]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.933559]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.933564]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.933569]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.933575]  [<ffffffff810923e6>] ? virtinfo_notifier_call+0x76/0xb0
Sep 16 13:10:26 node5 kernel: [29355.933609]  [<ffffffffa038733f>] ? zpl_read+0x10f/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933614]  [<ffffffff811b67b0>] open_exec+0x50/0x120
Sep 16 13:10:26 node5 kernel: [29355.933619]  [<ffffffff8120d8f8>] load_elf_binary+0x1628/0x1a10
Sep 16 13:10:26 node5 kernel: [29355.933624]  [<ffffffff81165853>] ? follow_page+0x413/0x500
Sep 16 13:10:26 node5 kernel: [29355.933629]  [<ffffffff8116c6e0>] ? __get_user_pages+0x110/0x410
Sep 16 13:10:26 node5 kernel: [29355.933634]  [<ffffffff8120990f>] ? load_misc_binary+0xaf/0x400
Sep 16 13:10:26 node5 kernel: [29355.933639]  [<ffffffff8116ca84>] ? get_user_pages+0x54/0x60
Sep 16 13:10:26 node5 kernel: [29355.933643]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.933649]  [<ffffffff811b60f7>] search_binary_handler+0x127/0x370
Sep 16 13:10:26 node5 kernel: [29355.933654]  [<ffffffff811b6aa3>] do_execve+0x223/0x2d0
Sep 16 13:10:26 node5 kernel: [29355.933660]  [<ffffffff8100966a>] sys_execve+0x4a/0x80
Sep 16 13:10:26 node5 kernel: [29355.933664]  [<ffffffff8100b55a>] stub_execve+0x6a/0xc0
Sep 16 13:10:26 node5 kernel: [29355.933668] exim          D ffff881bb4db2480     0 170948 148581 1297 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.933673]  ffff881bccae1638 0000000000000086 0000000000000286 0000000000000202
Sep 16 13:10:26 node5 kernel: [29355.933680]  ffff881bccae1668 ffffffffa02f9942 000000010100b172 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.933687]  ffff881bccae1608 ffffffff00000000 ffff881bb4db2a48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.933693] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.933721]  [<ffffffffa02f9942>] ? dmu_zfetch+0x4b2/0xe30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933731]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.933739]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.933762]  [<ffffffffa02d75fa>] arc_buf_add_ref+0x9a/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933786]  [<ffffffffa02e1946>] __dbuf_hold_impl+0x66/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933810]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933834]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933860]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933886]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933920]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933925]  [<ffffffff815326bb>] ? _spin_unlock_bh+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.933932]  [<ffffffffa02b4263>] ? nf_nat_hash_conntrack+0xf3/0x120 [nf_nat]
Sep 16 13:10:26 node5 kernel: [29355.933965]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.933999]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934032]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934064]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934097]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934102]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.934136]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934170]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934176]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.934181]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.934186]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.934191]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.934196]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.934201]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.934206]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.934211]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.934216]  [<ffffffff815326bb>] ? _spin_unlock_bh+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.934221]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.934226]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.934231]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.934236]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.934240]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.934245]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.934249] apache2       D ffff8839730ca140     0 170950 145855 2028 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.934255]  ffff8839ae9a9a88 0000000000000082 0000000000000000 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.934261]  ffff8839ae9a8000 000000000001b720 ffff8820737b0810 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.934268]  ffff8820737b0800 0000000101bb01cf ffff8839730ca708 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.934275] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.934280]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.934285]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.934290]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.934295]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.934300]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.934304]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.934310]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.934315]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.934320]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.934325]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.934331]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.934336]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.934340]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.934345]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.934351]  [<ffffffff8115bd05>] ? vma_prio_tree_add+0x75/0xd0
Sep 16 13:10:26 node5 kernel: [29355.934356]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.934360]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.934365]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.934370]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.934375]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.934380]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.934385]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.934389] httpd.itk     D ffff8819f5084880     0 170969 170255 1203 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.934394]  ffff881bccacf578 0000000000000082 000000010100869b ffffffffa0398340
Sep 16 13:10:26 node5 kernel: [29355.934400]  ffff88206e988400 0000000000000000 2029204400008290 ffff883b9fdcc3c0
Sep 16 13:10:26 node5 kernel: [29355.934407]  2e2022273d003834 64412720285f5f20 ffff8819f5084e48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.934414] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.934424]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.934431]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.934454]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934479]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934504]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934528]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934556]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934561]  [<ffffffff811c8a60>] ? d_callback+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.934588]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934614]  [<ffffffffa02e969a>] dmu_buf_hold_noread+0x3a/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934640]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934675]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934680]  [<ffffffff81142773>] ? __rmqueue+0xc3/0x490
Sep 16 13:10:26 node5 kernel: [29355.934714]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934719]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.934752]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934785]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934818]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934851]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934856]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.934890]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934924]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.934929]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.934933]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.934939]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.934943]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.934948]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.934953]  [<ffffffff810a7148>] ? up_read+0x18/0x30
Sep 16 13:10:26 node5 kernel: [29355.934957]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.934962]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.934967]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.934972]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.934976]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.934981]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.934987]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.934991] apache2       D ffff883c7ca9a900     0 170970 145855 2028 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.934996]  ffff883720ec7a88 0000000000000086 ffff880000034d88 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.935002]  ffffffff8100beae ffff883720ec7bb8 0000000000000200 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.935009]  0000000000000000 0000000000000000 ffff883c7ca9aec8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.935016] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.935020]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.935025]  [<ffffffff81146ab0>] ? __alloc_pages_nodemask+0x140/0xb20
Sep 16 13:10:26 node5 kernel: [29355.935031]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.935036]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.935041]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.935046]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.935051]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.935056]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.935062]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.935067]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.935072]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.935078]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.935083]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.935087]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.935093]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.935098]  [<ffffffff8115bd05>] ? vma_prio_tree_add+0x75/0xd0
Sep 16 13:10:26 node5 kernel: [29355.935103]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.935108]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.935113]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.935118]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.935123]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.935128]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.935133]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.935136] apache2       D ffff882e1c67f470     0 170971 145855 2028 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.935141]  ffff883bfebbba88 0000000000000082 0000000000000000 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.935148]  ffff883bfebba000 000000000001b720 ffff8820737b0810 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.935155]  ffff8820737b0800 0000000101bb05ac ffff882e1c67fa38 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.935161] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.935166]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.935171]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.935176]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.935181]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.935186]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.935192]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.935197]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.935202]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.935207]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.935213]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.935218]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.935222]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.935227]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.935232]  [<ffffffff8115bd05>] ? vma_prio_tree_add+0x75/0xd0
Sep 16 13:10:26 node5 kernel: [29355.935237]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.935242]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.935247]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.935252]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.935257]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.935262]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.935266]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.935270] httpd.itk     S ffff881d090110f0     0 170976 120341 1203 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.935275]  ffff881879e33cf8 0000000000000086 0000000000000000 00007fa8c78afe90
Sep 16 13:10:26 node5 kernel: [29355.935282]  ffff881879e33c78 ffffffff81193a10 ffff8820737b0810 ffff88207fc07380
Sep 16 13:10:26 node5 kernel: [29355.935289]  ffff8820737b0800 0000000101bb071f ffff881d090116b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.935295] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.935299]  [<ffffffff81193a10>] ? ub_slab_ptr+0x20/0x90
Sep 16 13:10:26 node5 kernel: [29355.935304]  [<ffffffff8123ff0d>] ? ipc_rcu_getref+0xd/0x10
Sep 16 13:10:26 node5 kernel: [29355.935309]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.935314]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.935318]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.935324]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.935329]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.935334]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.935339]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.935344]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.935348] sshd          S ffff8815bdba90b0     0 170977   4244    0 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.935353]  ffff8802987bb908 0000000000000086 0000000000000000 ffff880f280927c8
Sep 16 13:10:26 node5 kernel: [29355.935360]  0000000000000003 ffff884070f73020 ffff884070f3bd40 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.935367]  0000000000000000 0000000101bb8513 ffff8815bdba9678 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.935373] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.935379]  [<ffffffff815315ed>] schedule_hrtimeout_range+0x13d/0x160
Sep 16 13:10:26 node5 kernel: [29355.935384]  [<ffffffff8109b414>] ? flush_delayed_work+0x54/0x70
Sep 16 13:10:26 node5 kernel: [29355.935389]  [<ffffffff811c4cc9>] poll_schedule_timeout+0x39/0x60
Sep 16 13:10:26 node5 kernel: [29355.935394]  [<ffffffff811c5d8c>] do_select+0x57c/0x6c0
Sep 16 13:10:26 node5 kernel: [29355.935399]  [<ffffffff81489ff6>] ? __qdisc_run+0x96/0x120
Sep 16 13:10:26 node5 kernel: [29355.935405]  [<ffffffff811c4dc0>] ? __pollwait+0x0/0xf0
Sep 16 13:10:26 node5 kernel: [29355.935409]  [<ffffffff811c4eb0>] ? pollwake+0x0/0x60
Sep 16 13:10:26 node5 kernel: [29355.935413]  [<ffffffff811c4eb0>] ? pollwake+0x0/0x60
Sep 16 13:10:26 node5 kernel: [29355.935417]  [<ffffffff811c4eb0>] ? pollwake+0x0/0x60
Sep 16 13:10:26 node5 kernel: [29355.935421]  [<ffffffff811c4eb0>] ? pollwake+0x0/0x60
Sep 16 13:10:26 node5 kernel: [29355.935426]  [<ffffffff81457dfc>] ? sk_reset_timer+0x1c/0x30
Sep 16 13:10:26 node5 kernel: [29355.935432]  [<ffffffff814c0995>] ? tcp_event_new_data_sent+0xb5/0x110
Sep 16 13:10:26 node5 kernel: [29355.935437]  [<ffffffff814c0e17>] ? tcp_write_xmit+0x277/0xb20
Sep 16 13:10:26 node5 kernel: [29355.935442]  [<ffffffff815326bb>] ? _spin_unlock_bh+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.935447]  [<ffffffff814578f5>] ? release_sock+0xe5/0x110
Sep 16 13:10:26 node5 kernel: [29355.935452]  [<ffffffff814b3bd0>] ? tcp_sendmsg+0x6e0/0xb20
Sep 16 13:10:26 node5 kernel: [29355.935456]  [<ffffffff811c605a>] core_sys_select+0x18a/0x2d0
Sep 16 13:10:26 node5 kernel: [29355.935462]  [<ffffffff811c63f7>] sys_select+0x47/0x110
Sep 16 13:10:26 node5 kernel: [29355.935467]  [<ffffffff811ad2f1>] ? sys_write+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.935472]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.935476] /usr/sbin/mun D ffff881613a113f0     0 170999 166781  736 0x00000004
Sep 16 13:10:26 node5 kernel: [29355.935481]  ffff8805546cbb38 0000000000000082 000000000000000e 000000000000000e
Sep 16 13:10:26 node5 kernel: [29355.935488]  ffff8805546cbb28 ffffffff8114e385 ffffffff8100beae ffff8805546cbb98
Sep 16 13:10:26 node5 kernel: [29355.935494]  0000000000000200 0000000000000000 ffff881613a119b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.935501] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.935505]  [<ffffffff8114e385>] ? release_pages+0x255/0x3f0
Sep 16 13:10:26 node5 kernel: [29355.935510]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.935516]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.935520]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.935525]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.935530]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.935535]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.935540]  [<ffffffff8116b55e>] ? handle_pte_fault+0x2ce/0x1040
Sep 16 13:10:26 node5 kernel: [29355.935545]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.935550]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.935554]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.935559]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.935564]  [<ffffffff8116f211>] ? find_vma+0x21/0x80
Sep 16 13:10:26 node5 kernel: [29355.935568]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.935573]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.935578]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.935582]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.935587]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.935592]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.935596]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.935601]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.935606]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.935610] sshd          D ffff8815b7720fb0     0 171000  33896  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.935615]  ffff8808a9263748 0000000000000082 ffff8808a92636d8 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.935622]  ffff88207025c020 ffff882a44ba3680 ffff8808a92636d8 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.935628]  0000000000004030 ffff884071850000 ffff8815b7721578 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.935635] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.935669]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.935675]  [<ffffffff8146e672>] ? dev_hard_start_xmit+0x2e2/0x5f0
Sep 16 13:10:26 node5 kernel: [29355.935684]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.935691]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.935714]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.935739]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.935764]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.935788]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.935814]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.935842]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.935847]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.935882]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.935917]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.935922]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.935927]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.935960]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.935993]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.935998]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.936003]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.936007]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.936012]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.936018]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.936022]  [<ffffffff815324d2>] ? _spin_lock+0x12/0x30
Sep 16 13:10:26 node5 kernel: [29355.936027]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.936033]  [<ffffffff8118c3ea>] ? alloc_pages_current+0xaa/0x110
Sep 16 13:10:26 node5 kernel: [29355.936038]  [<ffffffff8104eb17>] ? pte_alloc_one+0x37/0x50
Sep 16 13:10:26 node5 kernel: [29355.936042]  [<ffffffff811679a2>] ? __pte_alloc+0xd2/0x350
Sep 16 13:10:26 node5 kernel: [29355.936047]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.936052]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.936057]  [<ffffffff811cad83>] ? __destroy_inode+0x33/0xb0
Sep 16 13:10:26 node5 kernel: [29355.936061]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.936066]  [<ffffffff811c8168>] ? d_free+0x58/0x60
Sep 16 13:10:26 node5 kernel: [29355.936070]  [<ffffffff811c81c4>] ? d_kill+0x54/0x70
Sep 16 13:10:26 node5 kernel: [29355.936075]  [<ffffffff811d0cd0>] ? mntput_no_expire+0x30/0x110
Sep 16 13:10:26 node5 kernel: [29355.936080]  [<ffffffff811ae82c>] ? __fput+0x1ec/0x280
Sep 16 13:10:26 node5 kernel: [29355.936085]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.936089]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.936093] bash          R ffff8815b77204c0     0 171001 170977    0 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.936098]  ffff88376ea68300 0000000900000000 ffffffff00000000 ffff8815b77204c0
Sep 16 13:10:26 node5 kernel: [29355.936105]  00007fff91a5a000 00007fff91a5a000 ffff88043445a468 00007fff91a44000
Sep 16 13:10:26 node5 kernel: [29355.936112]  ffff88331b367468 ffff880ec7e07d10 ffffffff81169fdc 00007fff91a44000
Sep 16 13:10:26 node5 kernel: [29355.936118] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.936123]  [<ffffffff81169fdc>] ? __copy_page_range+0x2bc/0x500
Sep 16 13:10:26 node5 kernel: [29355.936128]  [<ffffffff810b71ab>] ? __ub_kmem_uncharge+0xdb/0xe0
Sep 16 13:10:26 node5 kernel: [29355.936134]  [<ffffffff8128d391>] ? cpumask_any_but+0x31/0x50
Sep 16 13:10:26 node5 kernel: [29355.936139]  [<ffffffff81072e0a>] ? dup_mm+0x62a/0x8e0
Sep 16 13:10:26 node5 kernel: [29355.936145]  [<ffffffff810dc0d5>] ? cgroup_fork_callbacks+0x35/0x50
Sep 16 13:10:26 node5 kernel: [29355.936149]  [<ffffffff815324be>] ? _write_lock_irq+0x1e/0x20
Sep 16 13:10:26 node5 kernel: [29355.936154]  [<ffffffff81073c59>] ? copy_process+0xae9/0x1490
Sep 16 13:10:26 node5 kernel: [29355.936159]  [<ffffffff8119634e>] ? kmem_cache_alloc+0x1be/0x1e0
Sep 16 13:10:26 node5 kernel: [29355.936164]  [<ffffffff810746a9>] ? do_fork_pid+0xa9/0x490
Sep 16 13:10:26 node5 kernel: [29355.936169]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.936173]  [<ffffffff811a8e17>] ? fd_install+0x47/0x90
Sep 16 13:10:26 node5 kernel: [29355.936178]  [<ffffffff81074aaa>] ? do_fork+0x1a/0x20
Sep 16 13:10:26 node5 kernel: [29355.936183]  [<ffffffff810090f8>] ? sys_clone+0x28/0x30
Sep 16 13:10:26 node5 kernel: [29355.936188]  [<ffffffff8100b423>] ? stub_clone+0x13/0x20
Sep 16 13:10:26 node5 kernel: [29355.936193]  [<ffffffff8100b102>] ? system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.936197] sshd          D ffff8802c9c08680     0 171022  33896  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.936202]  ffff881649cf9748 0000000000000082 ffff881649cf96d8 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.936208]  ffff881649cf98d4 ffff88054db5eea0 ffff881649cf96d8 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.936215]  0000000000004030 ffff884071850000 ffff8802c9c08c48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.936222] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.936254]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.936264]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.936272]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.936295]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.936320]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.936345]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.936369]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.936395]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.936400]  [<ffffffff81194d1f>] ? cache_alloc_refill+0x9f/0x230
Sep 16 13:10:26 node5 kernel: [29355.936427]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.936433]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.936467]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.936500]  [<ffffffffa0371991>] ? zfs_getpage+0x1/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.936534]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.936539]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.936544]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.936577]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.936609]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.936614]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.936619]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.936624]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.936629]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.936634]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.936639]  [<ffffffff8118c3ea>] ? alloc_pages_current+0xaa/0x110
Sep 16 13:10:26 node5 kernel: [29355.936644]  [<ffffffff8104eb17>] ? pte_alloc_one+0x37/0x50
Sep 16 13:10:26 node5 kernel: [29355.936649]  [<ffffffff811679a2>] ? __pte_alloc+0xd2/0x350
Sep 16 13:10:26 node5 kernel: [29355.936654]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.936659]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.936664]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.936668]  [<ffffffff811c8168>] ? d_free+0x58/0x60
Sep 16 13:10:26 node5 kernel: [29355.936672]  [<ffffffff811c81c4>] ? d_kill+0x54/0x70
Sep 16 13:10:26 node5 kernel: [29355.936677]  [<ffffffff811d0cd0>] ? mntput_no_expire+0x30/0x110
Sep 16 13:10:26 node5 kernel: [29355.936682]  [<ffffffff811ae82c>] ? __fput+0x1ec/0x280
Sep 16 13:10:26 node5 kernel: [29355.936687]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.936692]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.936695] flush-zfs-38  S ffff8818910211b0     0 171030      2    0 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.936700]  ffff88036a57dda0 0000000000000046 0000000000000000 ffff88036a57dd78
Sep 16 13:10:26 node5 kernel: [29355.936707]  0000000101bb7d55 0000000000000400 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.936714]  0000000000000000 0000000000000024 ffff881891021778 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.936720] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.936725]  [<ffffffff81530902>] schedule_timeout+0x192/0x2e0
Sep 16 13:10:26 node5 kernel: [29355.936732]  [<ffffffff81088830>] ? process_timeout+0x0/0x10
Sep 16 13:10:26 node5 kernel: [29355.936737]  [<ffffffff811dcf3e>] bdi_writeback_task+0x14e/0x1e0
Sep 16 13:10:26 node5 kernel: [29355.936742]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.936746]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.936751]  [<ffffffff81162285>] bdi_start_fn+0x95/0x110
Sep 16 13:10:26 node5 kernel: [29355.936755]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.936760]  [<ffffffff810a15e6>] kthread+0x96/0xa0
Sep 16 13:10:26 node5 kernel: [29355.936765]  [<ffffffff8100c34a>] child_rip+0xa/0x20
Sep 16 13:10:26 node5 kernel: [29355.936770]  [<ffffffff810a1550>] ? kthread+0x0/0xa0
Sep 16 13:10:26 node5 kernel: [29355.936774]  [<ffffffff8100c340>] ? child_rip+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.936777] flush-zfs-81  S ffff8818910206c0     0 171031      2    0 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.936782]  ffff8804da5a5da0 0000000000000046 0000000000000000 ffff8804da5a5d78
Sep 16 13:10:26 node5 kernel: [29355.936789]  0000000101bb7d55 0000000000000400 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.936795]  0000000000000000 0000000101bb7d56 ffff881891020c88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.936802] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.936807]  [<ffffffff81530902>] schedule_timeout+0x192/0x2e0
Sep 16 13:10:26 node5 kernel: [29355.936812]  [<ffffffff81088830>] ? process_timeout+0x0/0x10
Sep 16 13:10:26 node5 kernel: [29355.936818]  [<ffffffff811dcf3e>] bdi_writeback_task+0x14e/0x1e0
Sep 16 13:10:26 node5 kernel: [29355.936822]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.936826]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.936831]  [<ffffffff81162285>] bdi_start_fn+0x95/0x110
Sep 16 13:10:26 node5 kernel: [29355.936835]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.936840]  [<ffffffff810a15e6>] kthread+0x96/0xa0
Sep 16 13:10:26 node5 kernel: [29355.936845]  [<ffffffff8100c34a>] child_rip+0xa/0x20
Sep 16 13:10:26 node5 kernel: [29355.936850]  [<ffffffff810a1550>] ? kthread+0x0/0xa0
Sep 16 13:10:26 node5 kernel: [29355.936854]  [<ffffffff8100c340>] ? child_rip+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.936857] httpd.itk     D ffff8812a4a05230     0 171032 170627 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.936862]  ffff88041d189b38 0000000000000082 0000000000000000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.936869]  00000000bae05d70 0000000000000002 ffffffff00000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.936876]  0000000000000000 0000000101bb1d1d ffff8812a4a057f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.936882] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.936887]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.936892]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.936897]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.936902]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.936907]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.936912]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.936917]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.936922]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.936926]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.936931]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.936936]  [<ffffffff8117014b>] ? vma_link+0x9b/0xf0
Sep 16 13:10:26 node5 kernel: [29355.936941]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.936946]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.936950]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.936955]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.936959]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.936965]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.936969] apache2       D ffff8836131394b0     0 171034 279139 2165 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.936974]  ffff8833a78e9b38 0000000000000082 ffff8833a78e9b00 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.936980]  00000000a78e9b28 0000000000d15380 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.936987]  ffff8820737b0800 0000000101bb1e36 ffff883613139a78 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.936994] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.936999]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.937004]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.937009]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.937013]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.937018]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.937023]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.937028]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.937033]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.937037]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.937042]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.937047]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.937053]  [<ffffffff8116f236>] ? find_vma+0x46/0x80
Sep 16 13:10:26 node5 kernel: [29355.937058]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.937063]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.937068]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.937073]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.937077]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.937082]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.937087]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.937091] httpd.itk     S ffff8812a4a04740     0 171035  73046 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.937096]  ffff88032e877cf8 0000000000000082 ffff88032e877cc0 00007ff4c9f28190
Sep 16 13:10:26 node5 kernel: [29355.937102]  ffff88032e877c78 000000000073c551 ffff882100c90770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.937109]  ffff882100c90760 0000000101bb1e4f ffff8812a4a04d08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.937116] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.937120]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.937126]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.937130]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.937136]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.937141]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.937146]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.937151]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.937156]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.937160] cron          D ffff883eb292ab30     0 171043 186015 2843 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.937165]  ffff883c81fbdad8 0000000000000086 ffff883c81fbda48 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.937172]  0000000000004030 ffff884071850000 ffff883c81fbdb68 ffffffffa01fe03a
Sep 16 13:10:26 node5 kernel: [29355.937179]  ffff883c81fbdaa8 ffffffffa03206cd ffff883eb292b0f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.937186] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.937219]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937231]  [<ffffffffa01fe03a>] ? spl_kmem_cache_alloc+0x9a/0x1090 [spl]
Sep 16 13:10:26 node5 kernel: [29355.937265]  [<ffffffffa03206cd>] ? sa_lookup+0x4d/0x60 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937269]  [<ffffffff815317d6>] ? down_read+0x16/0x30
Sep 16 13:10:26 node5 kernel: [29355.937279]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.937288]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.937310]  [<ffffffffa02d75fa>] arc_buf_add_ref+0x9a/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937335]  [<ffffffffa02e1946>] __dbuf_hold_impl+0x66/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937360]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937384]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937410]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937439]  [<ffffffffa02ea8e5>] dmu_buf_hold_array+0x65/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937447]  [<ffffffffa002a9c8>] ? avl_add+0x38/0x50 [zavl]
Sep 16 13:10:26 node5 kernel: [29355.937473]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937508]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937512]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.937546]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937552]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.937556]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.937561]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.937566]  [<ffffffff811ad210>] ? sys_read+0x0/0x90
Sep 16 13:10:26 node5 kernel: [29355.937572]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.937576] cron          D ffff880c7b06b2f0     0 171044  67628  515 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.937581]  ffff8804d6c23678 0000000000000082 ffff8804d6c23640 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.937588]  0000000000000000 00000000005f5360 ffff88012be10770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.937595]  ffff88012be10760 0000000101bb2a38 ffff880c7b06b8b8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.937602] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.937606]  [<ffffffff81055979>] ? mutex_spin_on_owner+0x19/0xc0
Sep 16 13:10:26 node5 kernel: [29355.937616]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.937623]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.937646]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937671]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937696]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937720]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937749]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937773]  [<ffffffffa02e0149>] ? dbuf_rele_and_unlock+0x229/0x320 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937801]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937827]  [<ffffffffa02e9984>] dmu_bonus_hold+0x34/0x290 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937850]  [<ffffffffa02e0640>] ? dbuf_rele+0x30/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937883]  [<ffffffffa031ecde>] sa_buf_hold+0xe/0x10 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937917]  [<ffffffffa0376d9f>] zfs_zget+0x10f/0x1e0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937951]  [<ffffffffa034daae>] ? zap_lookup_norm+0xbe/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.937986]  [<ffffffffa0356ccc>] zfs_dirent_lock+0x47c/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938019]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938053]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938059]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.938093]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938127]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938132]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.938137]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.938142]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.938148]  [<ffffffff8129aefb>] ? strncpy_from_user+0x5b/0x90
Sep 16 13:10:26 node5 kernel: [29355.938153]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.938157]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.938162]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.938167]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.938172]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.938177]  [<ffffffff811ae55e>] ? alloc_file+0x2e/0xe0
Sep 16 13:10:26 node5 kernel: [29355.938182]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.938187]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.938191]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.938196]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.938201]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.938206]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.938211] cron          D ffff8829e8fe4940     0 171045  39987 1037 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.938216]  ffff883c0aed7ab8 0000000000000082 0000000000000000 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.938223]  0000000000000282 ffff88054db5ea70 ffff88012be50770 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.938230]  ffff88012be50760 0000000101bb2a68 ffff8829e8fe4f08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.938237] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.938270]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938276]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.938282]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.938305]  [<ffffffffa02dee8c>] dbuf_find+0xac/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938330]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938355]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938379]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938405]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938435]  [<ffffffffa02ea8e5>] dmu_buf_hold_array+0x65/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938468]  [<ffffffffa036659e>] ? zfs_range_proxify+0x9e/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938495]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938529]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938563]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938568]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.938573]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.938579]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.938583] munin-node    D ffff88169a5bb470     0 171051 130385 1535 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.938588]  ffff8816ad35b5e8 0000000000000086 ffff8837a41898a0 0000000000000202
Sep 16 13:10:26 node5 kernel: [29355.938595]  ffff8816ad35b618 ffffffffa02f97bb 0000000000010206 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.938601]  0000000000000018 ffffffff00000000 ffff88169a5bba38 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.938608] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.938635]  [<ffffffffa02f97bb>] ? dmu_zfetch+0x32b/0xe30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938645]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.938653]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.938676]  [<ffffffffa02d75fa>] arc_buf_add_ref+0x9a/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938700]  [<ffffffffa02e1946>] __dbuf_hold_impl+0x66/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938724]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938748]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938773]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938801]  [<ffffffffa02fa8d4>] ? dnode_rele+0x54/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938826]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938861]  [<ffffffffa034754f>] zap_get_leaf_byblk+0x4f/0x2c0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938895]  [<ffffffffa034782a>] zap_deref_leaf+0x6a/0x80 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938929]  [<ffffffffa0347f20>] fzap_lookup+0x60/0x120 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938962]  [<ffffffffa034dad1>] zap_lookup_norm+0xe1/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.938996]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939029]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939062]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939095]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939101]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.939135]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939169]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939174]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.939179]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.939183]  [<ffffffff81194b53>] ? cache_grow+0x1f3/0x320
Sep 16 13:10:26 node5 kernel: [29355.939188]  [<ffffffff811ba76b>] ? get_fs_root+0x2b/0x80
Sep 16 13:10:26 node5 kernel: [29355.939193]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.939198]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.939202]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.939207]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.939212]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.939217]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.939221]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.939226]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.939231]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.939236]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.939241] httpd.itk     D ffff880148e21330     0 171052 170774 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.939247]  ffff881a6e0e9578 0000000000000082 0000000001000011 00d000a800000000
Sep 16 13:10:26 node5 kernel: [29355.939254]  0000000000000004 0000000000000000 0000000000000000 0000000000000028
Sep 16 13:10:26 node5 kernel: [29355.939260]  ffffffffa0416d80 0000000000000000 ffff880148e218f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.939267] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.939277]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.939285]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.939308]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939333]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939358]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939382]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939410]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939438]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939464]  [<ffffffffa02e969a>] dmu_buf_hold_noread+0x3a/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939490]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939524]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939558]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939591]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939624]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939656]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939689]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939694]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.939728]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939762]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.939767]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.939772]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.939777]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.939782]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.939787]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.939792]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.939797]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.939802]  [<ffffffff8117014b>] ? vma_link+0x9b/0xf0
Sep 16 13:10:26 node5 kernel: [29355.939807]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.939811]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.939816]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.939821]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.939826]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.939831]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.939835] apache2       D ffff883726bea480     0 171053 844331 2977 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.939840]  ffff883965abdb38 0000000000000086 0000000000000000 000000000000000e
Sep 16 13:10:26 node5 kernel: [29355.939847]  ffff883965abdb28 ffffffff8114e385 ffffffff8100beae ffff883965abdb98
Sep 16 13:10:26 node5 kernel: [29355.939854]  0000000000000000 0000000101bb3098 ffff883726beaa48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.939861] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.939865]  [<ffffffff8114e385>] ? release_pages+0x255/0x3f0
Sep 16 13:10:26 node5 kernel: [29355.939869]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.939876]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.939880]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.939885]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.939890]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.939895]  [<ffffffff811bdeb5>] __link_path_walk+0x905/0x1090
Sep 16 13:10:26 node5 kernel: [29355.939900]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.939905]  [<ffffffff81298860>] ? sprintf+0x40/0x50
Sep 16 13:10:26 node5 kernel: [29355.939910]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.939914]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.939920]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.939924]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.939930]  [<ffffffff8128d525>] ? _atomic_dec_and_lock+0x55/0x80
Sep 16 13:10:26 node5 kernel: [29355.939935]  [<ffffffff8129aeea>] ? strncpy_from_user+0x4a/0x90
Sep 16 13:10:26 node5 kernel: [29355.939940]  [<ffffffff811bb295>] ? getname_flags+0x155/0x260
Sep 16 13:10:26 node5 kernel: [29355.939945]  [<ffffffff811cdb22>] ? alloc_fd+0x92/0x160
Sep 16 13:10:26 node5 kernel: [29355.939950]  [<ffffffff811a9329>] do_sys_open+0x69/0x140
Sep 16 13:10:26 node5 kernel: [29355.939954]  [<ffffffff811a9440>] sys_open+0x20/0x30
Sep 16 13:10:26 node5 kernel: [29355.939959]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.939963] httpd.itk     S ffff880c7b06a800     0 171054  73046 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.939968]  ffff8808670e5cf8 0000000000000082 0000000000000000 ffff8813759f9e48
Sep 16 13:10:26 node5 kernel: [29355.939975]  ffff8808670e5c78 ffffffff81193a10 ffff881e3a436100 ffff88207fc07380
Sep 16 13:10:26 node5 kernel: [29355.939982]  0000000000000000 0000000101bb3213 ffff880c7b06adc8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.939989] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.939992]  [<ffffffff81193a10>] ? ub_slab_ptr+0x20/0x90
Sep 16 13:10:26 node5 kernel: [29355.939998]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.940004]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.940008]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.940014]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.940019]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.940024]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.940029]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.940034]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.940038] /usr/sbin/mun D ffff8814c73c4240     0 171057 166781  736 0x00000004
Sep 16 13:10:26 node5 kernel: [29355.940043]  ffff8808670e3b38 0000000000000082 000000000000000e 000000000000000e
Sep 16 13:10:26 node5 kernel: [29355.940050]  ffff8808670e3b28 ffffffff8114e385 0000000000000001 0000000000000282
Sep 16 13:10:26 node5 kernel: [29355.940057]  ffff8808670e3ad8 ffffffff8128d391 ffff8814c73c4808 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.940063] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.940067]  [<ffffffff8114e385>] ? release_pages+0x255/0x3f0
Sep 16 13:10:26 node5 kernel: [29355.940072]  [<ffffffff8128d391>] ? cpumask_any_but+0x31/0x50
Sep 16 13:10:26 node5 kernel: [29355.940077]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.940082]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.940088]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.940093]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.940097]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.940103]  [<ffffffff8116b55e>] ? handle_pte_fault+0x2ce/0x1040
Sep 16 13:10:26 node5 kernel: [29355.940107]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.940112]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.940117]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.940122]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.940126]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.940131]  [<ffffffff811b2b04>] ? cp_new_stat+0xe4/0x100
Sep 16 13:10:26 node5 kernel: [29355.940135]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.940140]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.940144]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.940149]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.940154]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.940159]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.940163] apache2       D ffff883045b5f430     0 171058  50154  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.940168]  ffff883de0bf5b38 0000000000000082 0000000000000000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.940175]  0000000000000000 0000000000000002 ffffffff8100beae ffff883de0bf5b68
Sep 16 13:10:26 node5 kernel: [29355.940182]  0000000000000000 0000000101bb7ecb ffff883045b5f9f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.940188] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.940192]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.940198]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.940203]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.940208]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.940212]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.940217]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.940222]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.940227]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.940231]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.940236]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.940241]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.940245]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.940250]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.940255]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.940259]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.940264]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.940269]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.940273] nrpe          D ffff883e3997ab70     0 171061  30044  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.940278]  ffff883f9defbab8 0000000000000086 0000000000000000 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.940285]  0000000000000282 ffff882a44ba2e20 ffff883f9defba48 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.940291]  0000000000000000 0000000101bb3d4d ffff883e3997b138 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.940298] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.940332]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940341]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.940348]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.940371]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940396]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940421]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940445]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940471]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940500]  [<ffffffffa02ea8e5>] dmu_buf_hold_array+0x65/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940526]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940561]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940566]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.940600]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940605]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.940610]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.940616]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.940620] nrpe          D ffff883e3997a080     0 171063  30044  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.940625]  ffff88402e7f1ab8 0000000000000082 ffff88402e7f1a80 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.940632]  0000000000000286 00000000009cc964 ffff882100c50770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.940638]  ffff882100c50760 0000000101bb413e ffff883e3997a648 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.940645] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.940649]  [<ffffffff81532527>] ? _spin_unlock_irqrestore+0x17/0x20
Sep 16 13:10:26 node5 kernel: [29355.940659]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.940667]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.940690]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940715]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940740]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940764]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940789]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940819]  [<ffffffffa02ea8e5>] dmu_buf_hold_array+0x65/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940844]  [<ffffffffa02ea951>] dmu_read_uio+0x41/0xd0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940879]  [<ffffffffa0371658>] zfs_read+0x178/0x4b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940884]  [<ffffffff81172f13>] ? mmap_region+0x343/0x770
Sep 16 13:10:26 node5 kernel: [29355.940918]  [<ffffffffa03872d5>] zpl_read+0xa5/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.940923]  [<ffffffff811ad125>] vfs_read+0xb5/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.940928]  [<ffffffff811ad261>] sys_read+0x51/0x90
Sep 16 13:10:26 node5 kernel: [29355.940934]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.940938] couriertcpd   D ffff88016c28c100     0 171066  43386  952 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.940943]  ffff8810331ef328 0000000000000086 0000000000000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.940950]  ffff8810331ef2a8 ffffffff815317d6 ffff88012be10770 0000000000000202
Sep 16 13:10:26 node5 kernel: [29355.940957]  0000000000000000 0000000101bb4c56 ffff88016c28c6c8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.940963] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.940967]  [<ffffffff815317d6>] ? down_read+0x16/0x30
Sep 16 13:10:26 node5 kernel: [29355.940977]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.940984]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.941007]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941033]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941057]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941081]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941107]  [<ffffffffa02e96e7>] dmu_buf_hold_noread+0x87/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941133]  [<ffffffffa02e981b>] dmu_buf_hold+0x3b/0x90 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941167]  [<ffffffffa034bb7a>] zap_lockdir+0x5a/0x810 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941201]  [<ffffffffa034da3a>] zap_lookup_norm+0x4a/0x190 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941235]  [<ffffffffa034dc13>] zap_lookup+0x33/0x40 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941268]  [<ffffffffa0356c65>] zfs_dirent_lock+0x415/0x540 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941302]  [<ffffffffa0356f9b>] zfs_dirlook+0x8b/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941335]  [<ffffffffa0350f00>] ? zfs_zaccess+0xa0/0x2d0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941340]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.941373]  [<ffffffffa03710ce>] zfs_lookup+0x2ee/0x340 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941407]  [<ffffffffa0387ab8>] zpl_lookup+0x78/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941413]  [<ffffffff811bd04d>] do_lookup+0x1dd/0x270
Sep 16 13:10:26 node5 kernel: [29355.941417]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.941423]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.941427]  [<ffffffff811ba76b>] ? get_fs_root+0x2b/0x80
Sep 16 13:10:26 node5 kernel: [29355.941432]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.941437]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.941442]  [<ffffffff811ae432>] ? get_empty_filp+0xa2/0x1a0
Sep 16 13:10:26 node5 kernel: [29355.941447]  [<ffffffff811bffa4>] do_filp_open+0x104/0xc50
Sep 16 13:10:26 node5 kernel: [29355.941453]  [<ffffffff810923e6>] ? virtinfo_notifier_call+0x76/0xb0
Sep 16 13:10:26 node5 kernel: [29355.941486]  [<ffffffffa038733f>] ? zpl_read+0x10f/0x140 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941492]  [<ffffffff811b67b0>] open_exec+0x50/0x120
Sep 16 13:10:26 node5 kernel: [29355.941497]  [<ffffffff8120d8f8>] load_elf_binary+0x1628/0x1a10
Sep 16 13:10:26 node5 kernel: [29355.941502]  [<ffffffff81165853>] ? follow_page+0x413/0x500
Sep 16 13:10:26 node5 kernel: [29355.941507]  [<ffffffff8116c6e0>] ? __get_user_pages+0x110/0x410
Sep 16 13:10:26 node5 kernel: [29355.941512]  [<ffffffff8120990f>] ? load_misc_binary+0xaf/0x400
Sep 16 13:10:26 node5 kernel: [29355.941517]  [<ffffffff8116ca84>] ? get_user_pages+0x54/0x60
Sep 16 13:10:26 node5 kernel: [29355.941522]  [<ffffffff811b53bc>] ? get_arg_page+0x5c/0x100
Sep 16 13:10:26 node5 kernel: [29355.941527]  [<ffffffff812995bd>] ? copy_user_generic_string+0x2d/0x40
Sep 16 13:10:26 node5 kernel: [29355.941532]  [<ffffffff811b60f7>] search_binary_handler+0x127/0x370
Sep 16 13:10:26 node5 kernel: [29355.941537]  [<ffffffff811b6aa3>] do_execve+0x223/0x2d0
Sep 16 13:10:26 node5 kernel: [29355.941543]  [<ffffffff8100966a>] sys_execve+0x4a/0x80
Sep 16 13:10:26 node5 kernel: [29355.941547]  [<ffffffff8100b55a>] stub_execve+0x6a/0xc0
Sep 16 13:10:26 node5 kernel: [29355.941551] httpd.itk     D ffff8808671c1370     0 171067 170895 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.941556]  ffff8805e118bb38 0000000000000082 000000000000000e 000000000000000e
Sep 16 13:10:26 node5 kernel: [29355.941563]  ffff8805e118bb28 ffffffff8114e385 ffffffff00000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.941570]  ffff880000034d88 0000000200000040 ffff8808671c1938 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.941577] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.941580]  [<ffffffff8114e385>] ? release_pages+0x255/0x3f0
Sep 16 13:10:26 node5 kernel: [29355.941585]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.941591]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.941596]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.941601]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.941606]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.941613]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.941618]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.941623]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.941628]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.941632]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.941637]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.941642]  [<ffffffff8117014b>] ? vma_link+0x9b/0xf0
Sep 16 13:10:26 node5 kernel: [29355.941647]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.941652]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.941656]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.941661]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.941665]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.941671]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.941676] couriertcpd   D ffff880dd225c180     0 171068 334471 1193 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.941682]  ffff8805494bfa88 0000000000000086 0000000000000000 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.941688]  ffff8805494be000 000000000001b720 0000000000000041 0000000000000002
Sep 16 13:10:26 node5 kernel: [29355.941695]  0000000000000000 0000000101bb4fca ffff880dd225c748 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.941701] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.941706]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.941711]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.941716]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.941721]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.941726]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.941731]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.941737]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.941742]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.941748]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.941753]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.941758]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.941763]  [<ffffffff8118c3ea>] ? alloc_pages_current+0xaa/0x110
Sep 16 13:10:26 node5 kernel: [29355.941769]  [<ffffffff8104eb17>] ? pte_alloc_one+0x37/0x50
Sep 16 13:10:26 node5 kernel: [29355.941774]  [<ffffffff811679a2>] ? __pte_alloc+0xd2/0x350
Sep 16 13:10:26 node5 kernel: [29355.941779]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.941783]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.941789]  [<ffffffff81454193>] ? move_addr_to_user+0x93/0xb0
Sep 16 13:10:26 node5 kernel: [29355.941794]  [<ffffffff81454814>] ? sys_getsockname+0xd4/0xf0
Sep 16 13:10:26 node5 kernel: [29355.941799]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.941805]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.941809] httpd.itk     S ffff8808671c0880     0 171070  73046 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.941814]  ffff8803badfdcf8 0000000000000086 ffff8803badfdcc0 00007ff4c9f28190
Sep 16 13:10:26 node5 kernel: [29355.941821]  ffff8803badfdc78 0000000003914d79 ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.941828]  ffff8820737b0800 0000000101bb515a ffff8808671c0e48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.941834] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.941839]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.941845]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.941849]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.941855]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.941860]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.941865]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.941870]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.941875]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.941879] exim4         D ffff8812513927c0     0 171071 168154 2834 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.941884]  ffff8803bad79748 0000000000000086 0000000000000000 0000000000000246
Sep 16 13:10:26 node5 kernel: [29355.941891]  00000000e07caf82 ffff883a3b64b600 ffff8803bad796d8 ffffffffa0382537
Sep 16 13:10:26 node5 kernel: [29355.941898]  0000000000000000 0000000101bb5438 ffff881251392d88 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.941904] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.941938]  [<ffffffffa0382537>] ? zio_cons+0x47/0xb0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.941948]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.941956]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.941979]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942004]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942029]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942053]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942079]  [<ffffffffa02ea245>] dmu_buf_hold_array_by_dnode+0x115/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942107]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942112]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.942147]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942182]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942187]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.942192]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.942225]  [<ffffffffa0386d20>] ? zpl_readpages+0x0/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942258]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942263]  [<ffffffff8114d381>] ? read_cache_pages+0x1/0x140
Sep 16 13:10:26 node5 kernel: [29355.942295]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942300]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.942305]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.942310]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.942315]  [<ffffffff8114e8f1>] ? lru_cache_add_lru+0x21/0x40
Sep 16 13:10:26 node5 kernel: [29355.942320]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.942325]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.942330]  [<ffffffff8118c3ea>] ? alloc_pages_current+0xaa/0x110
Sep 16 13:10:26 node5 kernel: [29355.942335]  [<ffffffff8104eb17>] ? pte_alloc_one+0x37/0x50
Sep 16 13:10:26 node5 kernel: [29355.942340]  [<ffffffff811679a2>] ? __pte_alloc+0xd2/0x350
Sep 16 13:10:26 node5 kernel: [29355.942345]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.942349]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.942354]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.942359]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.942364]  [<ffffffff8108f411>] ? do_sigaction+0x91/0x1f0
Sep 16 13:10:26 node5 kernel: [29355.942369]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.942374]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.942377] apache2       D ffff883358f84080     0 171072  50154  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.942382]  ffff88284b435b38 0000000000000082 ffff88284b434000 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.942389]  0000000000000000 0000000000000002 ffff882073e02370 000280da00000000
Sep 16 13:10:26 node5 kernel: [29355.942396]  0000000000000282 0000000000000030 ffff883358f84648 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.942402] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.942407]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.942413]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.942418]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.942423]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.942427]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.942432]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.942437]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.942443]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.942447]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.942452]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.942457]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.942462]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.942467]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.942471]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.942476]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.942480]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.942486]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.942489] apache2       S ffff88069de4a240     0 171080  72054 1088 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.942495]  ffff8803bac5dcf8 0000000000000086 ffff8803cebf7908 00007fe7ba921910
Sep 16 13:10:26 node5 kernel: [29355.942501]  ffff8803bac5dc78 ffffffff81193a10 ffff883efd3e3400 ffff88207fc07380
Sep 16 13:10:26 node5 kernel: [29355.942508]  ffff8803bac5dca8 ffffffff810b764c ffff88069de4a808 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.942515] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.942518]  [<ffffffff81193a10>] ? ub_slab_ptr+0x20/0x90
Sep 16 13:10:26 node5 kernel: [29355.942524]  [<ffffffff810b764c>] ? ub_slab_charge+0xac/0x100
Sep 16 13:10:26 node5 kernel: [29355.942529]  [<ffffffff81240249>] ? ipc_lock+0x39/0x60
Sep 16 13:10:26 node5 kernel: [29355.942534]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.942539]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.942543]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.942548]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.942553]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.942558]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.942563]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.942567] sendmail-mta  D ffff880fefe7a180     0 171081 168664 2502 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.942572]  ffff8803bac9b4d8 0000000000000082 ffff8803bac9b4a0 ffff883a3b64a540
Sep 16 13:10:26 node5 kernel: [29355.942579]  ffffffffa0394e00 00000000007771d2 ffff8840734c2410 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.942586]  ffff8840734c2400 0000000101bb5b9b ffff880fefe7a748 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.942592] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.942603]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.942611]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.942633]  [<ffffffffa02d57c7>] buf_hash_find+0x97/0x130 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942656]  [<ffffffffa02dc547>] arc_read+0xb7/0x910 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942680]  [<ffffffffa02e160f>] ? dbuf_read+0x49f/0x770 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942692]  [<ffffffffa01fddab>] ? kmem_free_debug+0x4b/0x150 [spl]
Sep 16 13:10:26 node5 kernel: [29355.942716]  [<ffffffffa02e3379>] dbuf_prefetch+0x1b9/0x270 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942744]  [<ffffffffa02f90ae>] dmu_zfetch_dofetch+0xfe/0x170 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942772]  [<ffffffffa02f9c88>] dmu_zfetch+0x7f8/0xe30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942795]  [<ffffffffa02dca83>] ? arc_read+0x5f3/0x910 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942818]  [<ffffffffa02dbf80>] ? l2arc_read_done+0x0/0x3b0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942843]  [<ffffffffa02e1577>] dbuf_read+0x407/0x770 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942869]  [<ffffffffa02ea269>] dmu_buf_hold_array_by_dnode+0x139/0x530 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942897]  [<ffffffffa02eb277>] dmu_read+0x97/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942902]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.942938]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942972]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.942978]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.942982]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.943015]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.943048]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.943053]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.943058]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.943063]  [<ffffffff81137539>] filemap_fault+0x4f9/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.943068]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.943073]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.943078]  [<ffffffff8118c3ea>] ? alloc_pages_current+0xaa/0x110
Sep 16 13:10:26 node5 kernel: [29355.943083]  [<ffffffff8104eb17>] ? pte_alloc_one+0x37/0x50
Sep 16 13:10:26 node5 kernel: [29355.943088]  [<ffffffff811679a2>] ? __pte_alloc+0xd2/0x350
Sep 16 13:10:26 node5 kernel: [29355.943093]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.943098]  [<ffffffff8116f236>] ? find_vma+0x46/0x80
Sep 16 13:10:26 node5 kernel: [29355.943102]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.943108]  [<ffffffff810ff7b4>] ? __call_rcu+0xc4/0x160
Sep 16 13:10:26 node5 kernel: [29355.943113]  [<ffffffff810ff885>] ? call_rcu_sched+0x15/0x20
Sep 16 13:10:26 node5 kernel: [29355.943117]  [<ffffffff810ff89e>] ? call_rcu+0xe/0x10
Sep 16 13:10:26 node5 kernel: [29355.943122]  [<ffffffff810a95ac>] ? __put_cred+0x3c/0x60
Sep 16 13:10:26 node5 kernel: [29355.943127]  [<ffffffff810a9c25>] ? commit_creds+0x115/0x1e0
Sep 16 13:10:26 node5 kernel: [29355.943132]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.943137]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.943141] apache2       D ffff882788276eb0     0 171086  38646  359 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.943146]  ffff883f72393688 0000000000000082 ffff883f72393650 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.943153]  0000000000000000 0000000000e62161 ffff88012be50230 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.943160]  ffff88012be50220 0000000101bb6b5a ffff882788277478 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.943166] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.943176]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.943183]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.943207]  [<ffffffffa02dee3b>] dbuf_find+0x5b/0x100 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.943232]  [<ffffffffa02e1981>] __dbuf_hold_impl+0xa1/0x4a0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.943236]  [<ffffffff81160a49>] ? zone_statistics+0x99/0xc0
Sep 16 13:10:26 node5 kernel: [29355.943261]  [<ffffffffa02e1e06>] dbuf_hold_impl+0x86/0xc0 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.943285]  [<ffffffffa02e3030>] dbuf_hold+0x20/0x30 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.943313]  [<ffffffffa02fc838>] dnode_hold_impl+0xe8/0x560 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.943341]  [<ffffffffa02fccc9>] dnode_hold+0x19/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.943367]  [<ffffffffa02eb213>] dmu_read+0x33/0x180 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.943372]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.943407]  [<ffffffffa0371ac7>] zfs_getpage+0x137/0x210 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.943440]  [<ffffffffa0386f11>] zpl_readpage+0x31/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.943446]  [<ffffffff81136cae>] ? add_to_page_cache_lru+0x3e/0x50
Sep 16 13:10:26 node5 kernel: [29355.943451]  [<ffffffff8114d422>] read_cache_pages+0xa2/0x140
Sep 16 13:10:26 node5 kernel: [29355.943484]  [<ffffffffa0386ee0>] ? zpl_readpage+0x0/0x70 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.943516]  [<ffffffffa0386d3e>] zpl_readpages+0x1e/0x20 [zfs]
Sep 16 13:10:26 node5 kernel: [29355.943521]  [<ffffffff8114d1c6>] __do_page_cache_readahead+0x196/0x220
Sep 16 13:10:26 node5 kernel: [29355.943526]  [<ffffffff8114d271>] ra_submit+0x21/0x30
Sep 16 13:10:26 node5 kernel: [29355.943530]  [<ffffffff8114d635>] ondemand_readahead+0x115/0x240
Sep 16 13:10:26 node5 kernel: [29355.943535]  [<ffffffff8114d8d6>] page_cache_sync_readahead+0x86/0xa0
Sep 16 13:10:26 node5 kernel: [29355.943540]  [<ffffffff811375fb>] filemap_fault+0x5bb/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.943545]  [<ffffffff810a1b55>] ? __wake_up_bit+0x15/0x40
Sep 16 13:10:26 node5 kernel: [29355.943551]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.943556]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.943560]  [<ffffffff8107d2f7>] ? current_fs_time+0x27/0x30
Sep 16 13:10:26 node5 kernel: [29355.943566]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.943571]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.943576]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.943581]  [<ffffffff8117367d>] ? do_mmap_pgoff+0x33d/0x3a0
Sep 16 13:10:26 node5 kernel: [29355.943586]  [<ffffffff815310de>] ? mutex_lock+0x1e/0x50
Sep 16 13:10:26 node5 kernel: [29355.943591]  [<ffffffff811acb0b>] ? generic_file_llseek+0x5b/0x70
Sep 16 13:10:26 node5 kernel: [29355.943596]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.943601]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.943605] apache2       S ffff882e1c67e980     0 171090  50154  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.943610]  ffff882e26185ce8 0000000000000086 ffffea00f8757c90 000002008114e880
Sep 16 13:10:26 node5 kernel: [29355.943617]  ffff883e1d5f21f0 00007f488783e012 ffff883e00000028 0000000000000004
Sep 16 13:10:26 node5 kernel: [29355.943623]  00007f488783e000 ffffea007d48b2c0 ffff882e1c67ef48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.943630] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.943634]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.943640]  [<ffffffff81530985>] schedule_timeout+0x215/0x2e0
Sep 16 13:10:26 node5 kernel: [29355.943645]  [<ffffffff8104eb17>] ? pte_alloc_one+0x37/0x50
Sep 16 13:10:26 node5 kernel: [29355.943649]  [<ffffffff815325e6>] ? _spin_lock_bh+0x16/0x40
Sep 16 13:10:26 node5 kernel: [29355.943655]  [<ffffffff814ae3f1>] inet_csk_accept+0x1c1/0x240
Sep 16 13:10:26 node5 kernel: [29355.943660]  [<ffffffff810a1c00>] ? autoremove_wake_function+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.943666]  [<ffffffff814d6aad>] inet_accept+0x2d/0x120
Sep 16 13:10:26 node5 kernel: [29355.943671]  [<ffffffff81455716>] sys_accept4+0x146/0x290
Sep 16 13:10:26 node5 kernel: [29355.943676]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.943681]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.943686]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.943691]  [<ffffffff811ae2f9>] ? fget+0x39/0x80
Sep 16 13:10:26 node5 kernel: [29355.943696]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.943701]  [<ffffffff81455870>] sys_accept+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.943707]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.943710] apache2       S ffff88026f817430     0 171107  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.943715]  ffff8802c2ce7cf8 0000000000000086 ffff8802c2ce7cc0 00007fab4572c250
Sep 16 13:10:26 node5 kernel: [29355.943722]  ffff8802c2ce7c78 00000000014a533a ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.943729]  ffff8820737b0800 0000000101bb7a5c ffff88026f8179f8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.943735] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.943741]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.943746]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.943751]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.943756]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.943761]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.943766]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.943771]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.943776]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.943780] httpd.itk     D ffff881935990780     0 171108 170899 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.943785]  ffff8802c2ce3b38 0000000000000082 ffff881935991068 000000000001b720
Sep 16 13:10:26 node5 kernel: [29355.943792]  00000000c2ce3ac8 0000000000000002 ffffffff00000000 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.943798]  ffff882080022c48 0000000200000040 ffff881935990d48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.943805] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.943809]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.943815]  [<ffffffff8153124e>] __mutex_lock_slowpath+0x13e/0x180
Sep 16 13:10:26 node5 kernel: [29355.943820]  [<ffffffff811c7b17>] ? __d_lookup+0xa7/0x150
Sep 16 13:10:26 node5 kernel: [29355.943825]  [<ffffffff815310eb>] mutex_lock+0x2b/0x50
Sep 16 13:10:26 node5 kernel: [29355.943830]  [<ffffffff811bcfc3>] do_lookup+0x153/0x270
Sep 16 13:10:26 node5 kernel: [29355.943837]  [<ffffffff811bd812>] __link_path_walk+0x262/0x1090
Sep 16 13:10:26 node5 kernel: [29355.943842]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.943847]  [<ffffffff811be8ba>] path_walk+0x6a/0xe0
Sep 16 13:10:26 node5 kernel: [29355.943852]  [<ffffffff811beacb>] filename_lookup+0x6b/0xc0
Sep 16 13:10:26 node5 kernel: [29355.943856]  [<ffffffff811bfbf9>] user_path_at+0x59/0xa0
Sep 16 13:10:26 node5 kernel: [29355.943861]  [<ffffffff8116f226>] ? find_vma+0x36/0x80
Sep 16 13:10:26 node5 kernel: [29355.943866]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.943871]  [<ffffffff8117014b>] ? vma_link+0x9b/0xf0
Sep 16 13:10:26 node5 kernel: [29355.943876]  [<ffffffff811b2990>] vfs_fstatat+0x50/0xa0
Sep 16 13:10:26 node5 kernel: [29355.943880]  [<ffffffff811b2a1b>] vfs_stat+0x1b/0x20
Sep 16 13:10:26 node5 kernel: [29355.943885]  [<ffffffff811b2be4>] sys_newstat+0x24/0x50
Sep 16 13:10:26 node5 kernel: [29355.943890]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.943894]  [<ffffffff815329e5>] ? page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.943899]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.943903] httpd.itk     S ffff88169a5ba980     0 171109  73046 1171 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.943908]  ffff8803bad93cf8 0000000000000082 ffff88067dfa5940 00007ff4c9f28190
Sep 16 13:10:26 node5 kernel: [29355.943915]  ffff8803bad93c78 ffffffff81193a10 ffff881e3a436100 ffff88207fc07380
Sep 16 13:10:26 node5 kernel: [29355.943921]  ffff8803bad93ca8 ffffffff810b764c ffff88169a5baf48 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.943928] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.943932]  [<ffffffff81193a10>] ? ub_slab_ptr+0x20/0x90
Sep 16 13:10:26 node5 kernel: [29355.943937]  [<ffffffff810b764c>] ? ub_slab_charge+0xac/0x100
Sep 16 13:10:26 node5 kernel: [29355.943942]  [<ffffffff81240249>] ? ipc_lock+0x39/0x60
Sep 16 13:10:26 node5 kernel: [29355.943947]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.943952]  [<ffffffff810a7148>] ? up_read+0x18/0x30
Sep 16 13:10:26 node5 kernel: [29355.943956]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.943962]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.943967]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.943972]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.943977]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.943982]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.943986] flush-zfs-17  S ffff880148e20840     0 171110      2    0 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.943991]  ffff8803bad97da0 0000000000000046 000000000000285d 0000000000000000
Sep 16 13:10:26 node5 kernel: [29355.943997]  ffff8803bad97d40 ffffffff8106abb1 ffff8803bad97d40 ffff880148e20840
Sep 16 13:10:26 node5 kernel: [29355.944004]  ffff884073408cb0 ffffffff8160c900 ffff880148e20e08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.944011] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.944015]  [<ffffffff8106abb1>] ? update_curr+0xe1/0x1f0
Sep 16 13:10:26 node5 kernel: [29355.944020]  [<ffffffff81530902>] schedule_timeout+0x192/0x2e0
Sep 16 13:10:26 node5 kernel: [29355.944026]  [<ffffffff81088830>] ? process_timeout+0x0/0x10
Sep 16 13:10:26 node5 kernel: [29355.944031]  [<ffffffff811dcf3e>] bdi_writeback_task+0x14e/0x1e0
Sep 16 13:10:26 node5 kernel: [29355.944036]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.944040]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.944044]  [<ffffffff81162285>] bdi_start_fn+0x95/0x110
Sep 16 13:10:26 node5 kernel: [29355.944049]  [<ffffffff811621f0>] ? bdi_start_fn+0x0/0x110
Sep 16 13:10:26 node5 kernel: [29355.944054]  [<ffffffff810a15e6>] kthread+0x96/0xa0
Sep 16 13:10:26 node5 kernel: [29355.944059]  [<ffffffff8100c34a>] child_rip+0xa/0x20
Sep 16 13:10:26 node5 kernel: [29355.944064]  [<ffffffff810a1550>] ? kthread+0x0/0xa0
Sep 16 13:10:26 node5 kernel: [29355.944068]  [<ffffffff8100c340>] ? child_rip+0x0/0x20
Sep 16 13:10:26 node5 kernel: [29355.944071] apache2       S ffff882e3aaa4cb0     0 171116  50154  978 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.944076]  ffff883f10059ce8 0000000000000086 ffffea00ba834750 000002008114e880
Sep 16 13:10:26 node5 kernel: [29355.944083]  ffff882ea0d1d1f0 00007f488783e012 ffff882e00000028 0000000000000004
Sep 16 13:10:26 node5 kernel: [29355.944090]  00007f488783e000 ffffea007d48b2c0 ffff882e3aaa5278 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.944096] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.944101]  [<ffffffff8116b389>] ? handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.944106]  [<ffffffff81530985>] schedule_timeout+0x215/0x2e0
Sep 16 13:10:26 node5 kernel: [29355.944111]  [<ffffffff8104eb17>] ? pte_alloc_one+0x37/0x50
Sep 16 13:10:26 node5 kernel: [29355.944115]  [<ffffffff815325e6>] ? _spin_lock_bh+0x16/0x40
Sep 16 13:10:26 node5 kernel: [29355.944120]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.944125]  [<ffffffff814ae3f1>] inet_csk_accept+0x1c1/0x240
Sep 16 13:10:26 node5 kernel: [29355.944131]  [<ffffffff810a1c00>] ? autoremove_wake_function+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.944135]  [<ffffffff814d6aad>] inet_accept+0x2d/0x120
Sep 16 13:10:26 node5 kernel: [29355.944140]  [<ffffffff81455716>] sys_accept4+0x146/0x290
Sep 16 13:10:26 node5 kernel: [29355.944145]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.944151]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.944156]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.944161]  [<ffffffff81455870>] sys_accept+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.944166]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b
Sep 16 13:10:26 node5 kernel: [29355.944170] couriertcpd   D ffff8815bdb58300     0 171118 334471 1193 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.944175]  ffff880420b13a88 0000000000000082 ffff880420b13a50 0000000000000001
Sep 16 13:10:26 node5 kernel: [29355.944182]  ffff880420b12000 0000000000fde9ac ffff882100c90770 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.944189]  ffff882100c90760 0000000101bb80e1 ffff8815bdb588c8 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.944196] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.944200]  [<ffffffff81134ff0>] ? sync_page_killable+0x0/0x40
Sep 16 13:10:26 node5 kernel: [29355.944206]  [<ffffffff8152fd13>] io_schedule+0x73/0xc0
Sep 16 13:10:26 node5 kernel: [29355.944210]  [<ffffffff81134fdd>] sync_page+0x3d/0x50
Sep 16 13:10:26 node5 kernel: [29355.944215]  [<ffffffff81134ffe>] sync_page_killable+0xe/0x40
Sep 16 13:10:26 node5 kernel: [29355.944220]  [<ffffffff81530d3f>] __wait_on_bit+0x5f/0x90
Sep 16 13:10:26 node5 kernel: [29355.944226]  [<ffffffff81135195>] wait_on_page_bit_killable+0x75/0x80
Sep 16 13:10:26 node5 kernel: [29355.944231]  [<ffffffff810a1c80>] ? wake_bit_function+0x0/0x50
Sep 16 13:10:26 node5 kernel: [29355.944236]  [<ffffffff8113532a>] __lock_page_or_retry+0xaa/0xc0
Sep 16 13:10:26 node5 kernel: [29355.944242]  [<ffffffff81137267>] filemap_fault+0x227/0x5d0
Sep 16 13:10:26 node5 kernel: [29355.944246]  [<ffffffff811341d1>] ? page_waitqueue+0x71/0x90
Sep 16 13:10:26 node5 kernel: [29355.944251]  [<ffffffff8116acce>] __do_fault+0x7e/0x640
Sep 16 13:10:26 node5 kernel: [29355.944257]  [<ffffffff8116b389>] handle_pte_fault+0xf9/0x1040
Sep 16 13:10:26 node5 kernel: [29355.944262]  [<ffffffff8118c3ea>] ? alloc_pages_current+0xaa/0x110
Sep 16 13:10:26 node5 kernel: [29355.944267]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.944272]  [<ffffffff8116c4fa>] handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.944276]  [<ffffffff8104a7a8>] __do_page_fault+0x138/0x480
Sep 16 13:10:26 node5 kernel: [29355.944282]  [<ffffffff81454193>] ? move_addr_to_user+0x93/0xb0
Sep 16 13:10:26 node5 kernel: [29355.944287]  [<ffffffff81454814>] ? sys_getsockname+0xd4/0xf0
Sep 16 13:10:26 node5 kernel: [29355.944291]  [<ffffffff8153561e>] do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.944296]  [<ffffffff815329e5>] page_fault+0x25/0x30
Sep 16 13:10:26 node5 kernel: [29355.944300] apache2       S ffff88026f816940     0 171119  34335  229 0x00000000
Sep 16 13:10:26 node5 kernel: [29355.944305]  ffff88026f989cf8 0000000000000082 ffff88026f989cc0 00007fab4572c250
Sep 16 13:10:26 node5 kernel: [29355.944311]  ffff88026f989c78 000000000102b9df ffff8820737b0810 ffff882000000000
Sep 16 13:10:26 node5 kernel: [29355.944318]  ffff8820737b0800 0000000101bb8249 ffff88026f816f08 000000000001ec80
Sep 16 13:10:26 node5 kernel: [29355.944325] Call Trace:
Sep 16 13:10:26 node5 kernel: [29355.944329]  [<ffffffff81244c21>] sys_semtimedop+0x811/0x830
Sep 16 13:10:26 node5 kernel: [29355.944334]  [<ffffffff8100beae>] ? call_function_interrupt+0xe/0x20
Sep 16 13:10:26 node5 kernel: [29355.944339]  [<ffffffff8116c4fa>] ? handle_mm_fault+0x22a/0x300
Sep 16 13:10:26 node5 kernel: [29355.944344]  [<ffffffff8104a85c>] ? __do_page_fault+0x1ec/0x480
Sep 16 13:10:26 node5 kernel: [29355.944349]  [<ffffffff811f983c>] ? ep_insert+0x32c/0x390
Sep 16 13:10:26 node5 kernel: [29355.944354]  [<ffffffff811f8660>] ? ep_ptable_queue_proc+0x0/0xb0
Sep 16 13:10:26 node5 kernel: [29355.944359]  [<ffffffff8153561e>] ? do_page_fault+0x3e/0xa0
Sep 16 13:10:26 node5 kernel: [29355.944364]  [<ffffffff81244c50>] sys_semop+0x10/0x20
Sep 16 13:10:26 node5 kernel: [29355.944369]  [<ffffffff8100b102>] system_call_fastpath+0x16/0x1b