Skip to content

Instantly share code, notes, and snippets.

@pstch
Created December 19, 2016 00:39
Show Gist options
  • Save pstch/62e9dea14fb2fa97dcacaccca2eead80 to your computer and use it in GitHub Desktop.
Save pstch/62e9dea14fb2fa97dcacaccca2eead80 to your computer and use it in GitHub Desktop.
[139741.315562] VERIFY3(0 == hdr->b_l1hdr.b_bufcnt) failed (0 == 1)
[139741.316428] VERIFY3(0 == hdr->b_l1hdr.b_bufcnt) failed (0 == 1)
[139741.316429] PANIC at arc.c:5347:arc_read_done()
[139741.316430] Showing stack for process 270
[139741.316433] CPU: 0 PID: 270 Comm: z_rd_int_7 Tainted: P O 3.16.0-4-amd64 #1 Debian 3.16.36-1+deb8u1
[139741.316434] Hardware name: Dell Inc. PowerEdge 1950/0UR033, BIOS 2.7.0 10/30/2010
[139741.316438] 0000000000000000 ffffffff81512391 ffffffffa0467b65 ffff88039abd3d30
[139741.316440] ffffffffa019b9ff ffffffffa02744e3 ffff880300000030 ffff88039abd3d40
[139741.316441] ffff88039abd3ce0 2833594649524556 726468203d3d2030 6468316c5f623e2d
[139741.316442] Call Trace:
[139741.316450] [<ffffffff81512391>] ? dump_stack+0x5d/0x78
[139741.316472] [<ffffffffa019b9ff>] ? spl_panic+0xbf/0xf0 [spl]
[139741.316478] [<ffffffffa02744e3>] ? fletcher_4_native+0xb3/0x140 [zcommon]
[139741.316483] [<ffffffffa0274570>] ? fletcher_4_native+0x140/0x140 [zcommon]
[139741.316533] [<ffffffffa02d6ea8>] ? abd_iterate_func+0x118/0x3b0 [zfs]
[139741.316551] [<ffffffffa02d4250>] ? abd_copy_from_buf_off_cb+0x20/0x20 [zfs]
[139741.316570] [<ffffffffa02d783d>] ? abd_iterate_func2+0x1cd/0x720 [zfs]
[139741.316574] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139741.316594] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139741.316613] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139741.316636] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139741.316658] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139741.316663] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139741.316667] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139741.316672] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139741.316675] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139741.316678] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139741.316680] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139741.316682] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139741.338440] PANIC at arc.c:5347:arc_read_done()
[139741.338521] VERIFY3(0 == hdr->b_l1hdr.b_bufcnt) failed (0 == 1)
[139741.338523] PANIC at arc.c:5347:arc_read_done()
[139741.338524] Showing stack for process 263
[139741.338526] CPU: 2 PID: 263 Comm: z_rd_int_0 Tainted: P O 3.16.0-4-amd64 #1 Debian 3.16.36-1+deb8u1
[139741.338527] Hardware name: Dell Inc. PowerEdge 1950/0UR033, BIOS 2.7.0 10/30/2010
[139741.338530] 0000000000000000 ffffffff81512391 ffffffffa0467b65 ffff88039a7abd30
[139741.338532] ffffffffa019b9ff ffffffffa02744e3 ffff880300000030 ffff88039a7abd40
[139741.338533] ffff88039a7abce0 2833594649524556 726468203d3d2030 6468316c5f623e2d
[139741.338534] Call Trace:
[139741.338537] [<ffffffff81512391>] ? dump_stack+0x5d/0x78
[139741.338546] [<ffffffffa019b9ff>] ? spl_panic+0xbf/0xf0 [spl]
[139741.338551] [<ffffffffa02744e3>] ? fletcher_4_native+0xb3/0x140 [zcommon]
[139741.338558] [<ffffffffa0274570>] ? fletcher_4_native+0x140/0x140 [zcommon]
[139741.338563] [<ffffffff810a9174>] ? __wake_up+0x34/0x50
[139741.338572] [<ffffffffa01993a0>] ? taskq_thread_create+0x100/0x100 [spl]
[139741.338580] [<ffffffffa019823c>] ? taskq_dispatch+0x6c/0x2d0 [spl]
[139741.338606] [<ffffffffa03d13e0>] ? zio_taskq_member.isra.7.constprop.10+0x80/0x80 [zfs]
[139741.338614] [<ffffffffa01984ea>] ? taskq_thread_spawn+0x4a/0x50 [spl]
[139741.338620] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139741.338643] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139741.338664] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139741.338687] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139741.338712] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139741.338719] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139741.338724] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139741.338731] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139741.338734] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139741.338736] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139741.338738] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139741.338740] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139741.338801] VERIFY3(0 == hdr->b_l1hdr.b_bufcnt) failed (0 == 1)
[139741.338803] PANIC at arc.c:5347:arc_read_done()
[139741.338804] Showing stack for process 268
[139741.338806] CPU: 0 PID: 268 Comm: z_rd_int_5 Tainted: P O 3.16.0-4-amd64 #1 Debian 3.16.36-1+deb8u1
[139741.338807] Hardware name: Dell Inc. PowerEdge 1950/0UR033, BIOS 2.7.0 10/30/2010
[139741.338809] 0000000000000000 ffffffff81512391 ffffffffa0467b65 ffff88039abcbd30
[139741.338811] ffffffffa019b9ff 0000000000000046 ffff880300000030 ffff88039abcbd40
[139741.338813] ffff88039abcbce0 2833594649524556 726468203d3d2030 6468316c5f623e2d
[139741.338813] Call Trace:
[139741.338815] [<ffffffff81512391>] ? dump_stack+0x5d/0x78
[139741.338823] [<ffffffffa019b9ff>] ? spl_panic+0xbf/0xf0 [spl]
[139741.338828] [<ffffffffa0193fcf>] ? spl_kmem_alloc+0xef/0x1a0 [spl]
[139741.338830] [<ffffffff810a9174>] ? __wake_up+0x34/0x50
[139741.338835] [<ffffffffa01993a0>] ? taskq_thread_create+0x100/0x100 [spl]
[139741.338840] [<ffffffffa019823c>] ? taskq_dispatch+0x6c/0x2d0 [spl]
[139741.338841] VERIFY3(0 == hdr->b_l1hdr.b_bufcnt) failed (0 == 1)
[139741.338843] PANIC at arc.c:5347:arc_read_done()
[139741.338864] [<ffffffffa03d13e0>] ? zio_taskq_member.isra.7.constprop.10+0x80/0x80 [zfs]
[139741.338865] Showing stack for process 267
[139741.338870] [<ffffffffa01984ea>] ? taskq_thread_spawn+0x4a/0x50 [spl]
[139741.338872] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139741.338892] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139741.338911] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139741.338934] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139741.338956] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139741.338961] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139741.338963] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139741.338968] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139741.338970] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139741.338972] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139741.338974] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139741.338976] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139741.338979] CPU: 2 PID: 267 Comm: z_rd_int_4 Tainted: P O 3.16.0-4-amd64 #1 Debian 3.16.36-1+deb8u1
[139741.338980] Hardware name: Dell Inc. PowerEdge 1950/0UR033, BIOS 2.7.0 10/30/2010
[139741.338982] 0000000000000000 ffffffff81512391 ffffffffa0467b65 ffff88039afffd30
[139741.338984] ffffffffa019b9ff 0000000000000046 ffff880000000030 ffff88039afffd40
[139741.338987] ffff88039afffce0 2833594649524556 726468203d3d2030 6468316c5f623e2d
[139741.338988] Call Trace:
[139741.338990] [<ffffffff81512391>] ? dump_stack+0x5d/0x78
[139741.338997] [<ffffffffa019b9ff>] ? spl_panic+0xbf/0xf0 [spl]
[139741.339002] [<ffffffffa0193fcf>] ? spl_kmem_alloc+0xef/0x1a0 [spl]
[139741.339004] [<ffffffff810a9174>] ? __wake_up+0x34/0x50
[139741.339009] [<ffffffffa01993a0>] ? taskq_thread_create+0x100/0x100 [spl]
[139741.339014] [<ffffffffa019823c>] ? taskq_dispatch+0x6c/0x2d0 [spl]
[139741.339036] [<ffffffffa03d13e0>] ? zio_taskq_member.isra.7.constprop.10+0x80/0x80 [zfs]
[139741.339043] [<ffffffffa01984ea>] ? taskq_thread_spawn+0x4a/0x50 [spl]
[139741.339045] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139741.339064] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139741.339084] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139741.339106] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139741.339128] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139741.339135] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139741.339137] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139741.339142] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139741.339144] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139741.339146] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139741.339149] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139741.339151] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139741.339693] VERIFY3(0 == hdr->b_l1hdr.b_bufcnt) failed (0 == 1)
[139741.339694] PANIC at arc.c:5347:arc_read_done()
[139741.339695] Showing stack for process 2264
[139741.339698] CPU: 1 PID: 2264 Comm: z_rd_int_7 Tainted: P O 3.16.0-4-amd64 #1 Debian 3.16.36-1+deb8u1
[139741.339699] Hardware name: Dell Inc. PowerEdge 1950/0UR033, BIOS 2.7.0 10/30/2010
[139741.339703] 0000000000000000 ffffffff81512391 ffffffffa0467b65 ffff880357c1bd30
[139741.339705] ffffffffa019b9ff ffffffffa02744e3 ffff880300000030 ffff880357c1bd40
[139741.339706] ffff880357c1bce0 2833594649524556 726468203d3d2030 6468316c5f623e2d
[139741.339707] Call Trace:
[139741.339715] [<ffffffff81512391>] ? dump_stack+0x5d/0x78
[139741.339740] [<ffffffffa019b9ff>] ? spl_panic+0xbf/0xf0 [spl]
[139741.339746] [<ffffffffa02744e3>] ? fletcher_4_native+0xb3/0x140 [zcommon]
[139741.339751] [<ffffffffa0274570>] ? fletcher_4_native+0x140/0x140 [zcommon]
[139741.339810] [<ffffffffa02d6ea8>] ? abd_iterate_func+0x118/0x3b0 [zfs]
[139741.339816] [<ffffffff812b67aa>] ? number.isra.2+0x31a/0x350
[139741.339835] [<ffffffffa02d783d>] ? abd_iterate_func2+0x1cd/0x720 [zfs]
[139741.339838] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139741.339858] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139741.339879] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139741.339902] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139741.339927] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139741.339933] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139741.339937] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139741.339942] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139741.339947] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139741.339949] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139741.339952] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139741.339954] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139741.340584] VERIFY3(0 == hdr->b_l1hdr.b_bufcnt) failed (0 == 1)
[139741.340585] PANIC at arc.c:5347:arc_read_done()
[139741.420012] VERIFY3(0 == hdr->b_l1hdr.b_bufcnt) failed (0 == 1)
[139741.420013] PANIC at arc.c:5347:arc_read_done()
[139741.420179] VERIFY3(0 == hdr->b_l1hdr.b_bufcnt) failed (0 == 1)
[139741.420180] PANIC at arc.c:5347:arc_read_done()
[139741.420185] VERIFY3(0 == hdr->b_l1hdr.b_bufcnt) failed (0 == 1)
[139741.420186] PANIC at arc.c:5347:arc_read_done()
[139920.216023] INFO: task z_rd_int_0:263 blocked for more than 120 seconds.
[139920.222562] Tainted: P O 3.16.0-4-amd64 #1
[139920.229209] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[139920.235944] z_rd_int_0 D ffff880035fa0e78 0 263 2 0x00000000
[139920.235949] ffff880035fa0a20 0000000000000046 0000000000012f40 ffff88039a7abfd8
[139920.235953] 0000000000012f40 ffff880035fa0a20 ffffffffa0467b65 ffff88039a7abd30
[139920.235956] 00000000000014e3 ffffffffa045183f ffffffffa0468ee0 0000000000000000
[139920.235960] Call Trace:
[139920.235987] [<ffffffffa019ba25>] ? spl_panic+0xe5/0xf0 [spl]
[139920.235995] [<ffffffffa0274570>] ? fletcher_4_native+0x140/0x140 [zcommon]
[139920.236001] [<ffffffff810a9174>] ? __wake_up+0x34/0x50
[139920.236013] [<ffffffffa01993a0>] ? taskq_thread_create+0x100/0x100 [spl]
[139920.236021] [<ffffffffa019823c>] ? taskq_dispatch+0x6c/0x2d0 [spl]
[139920.236071] [<ffffffffa03d13e0>] ? zio_taskq_member.isra.7.constprop.10+0x80/0x80 [zfs]
[139920.236079] [<ffffffffa01984ea>] ? taskq_thread_spawn+0x4a/0x50 [spl]
[139920.236084] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139920.236105] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139920.236126] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139920.236150] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139920.236173] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139920.236180] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139920.236184] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139920.236190] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139920.236194] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139920.236198] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.236201] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139920.236204] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.236207] INFO: task z_rd_int_1:264 blocked for more than 120 seconds.
[139920.243014] Tainted: P O 3.16.0-4-amd64 #1
[139920.249787] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[139920.256545] z_rd_int_1 D ffff880035f864e8 0 264 2 0x00000000
[139920.256549] ffff880035f86090 0000000000000046 0000000000012f40 ffff88039a7a3fd8
[139920.256552] 0000000000012f40 ffff880035f86090 ffffffffa0467b65 ffff88039a7a3d30
[139920.256555] 00000000000014e3 ffffffffa045183f ffffffffa0468ee0 0000000000000000
[139920.256558] Call Trace:
[139920.256570] [<ffffffffa019ba25>] ? spl_panic+0xe5/0xf0 [spl]
[139920.256575] [<ffffffff810a3851>] ? load_balance+0x191/0x850
[139920.256578] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139920.256600] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139920.256621] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139920.256626] [<ffffffff8101ca45>] ? sched_clock+0x5/0x10
[139920.256650] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139920.256673] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139920.256680] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139920.256683] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139920.256689] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139920.256692] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139920.256695] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.256699] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139920.256702] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.256705] INFO: task z_rd_int_4:267 blocked for more than 120 seconds.
[139920.263487] Tainted: P O 3.16.0-4-amd64 #1
[139920.270286] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[139920.277126] z_rd_int_4 D ffff880035f6c468 0 267 2 0x00000000
[139920.277130] ffff880035f6c010 0000000000000046 0000000000012f40 ffff88039affffd8
[139920.277135] 0000000000012f40 ffff880035f6c010 ffffffffa0467b65 ffff88039afffd30
[139920.277138] 00000000000014e3 ffffffffa045183f ffffffffa0468ee0 0000000000000000
[139920.277141] Call Trace:
[139920.277152] [<ffffffffa019ba25>] ? spl_panic+0xe5/0xf0 [spl]
[139920.277158] [<ffffffffa0193fcf>] ? spl_kmem_alloc+0xef/0x1a0 [spl]
[139920.277161] [<ffffffff810a9174>] ? __wake_up+0x34/0x50
[139920.277167] [<ffffffffa01993a0>] ? taskq_thread_create+0x100/0x100 [spl]
[139920.277173] [<ffffffffa019823c>] ? taskq_dispatch+0x6c/0x2d0 [spl]
[139920.277198] [<ffffffffa03d13e0>] ? zio_taskq_member.isra.7.constprop.10+0x80/0x80 [zfs]
[139920.277205] [<ffffffffa01984ea>] ? taskq_thread_spawn+0x4a/0x50 [spl]
[139920.277208] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139920.277230] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139920.277251] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139920.277275] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139920.277298] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139920.277305] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139920.277308] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139920.277315] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139920.277318] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139920.277321] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.277324] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139920.277327] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.277330] INFO: task z_rd_int_5:268 blocked for more than 120 seconds.
[139920.284198] Tainted: P O 3.16.0-4-amd64 #1
[139920.291126] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[139920.298144] z_rd_int_5 D ffff880035f6d708 0 268 2 0x00000000
[139920.298148] ffff880035f6d2b0 0000000000000046 0000000000012f40 ffff88039abcbfd8
[139920.298154] 0000000000012f40 ffff880035f6d2b0 ffffffffa0467b65 ffff88039abcbd30
[139920.298157] 00000000000014e3 ffffffffa045183f ffffffffa0468ee0 0000000000000000
[139920.298160] Call Trace:
[139920.298170] [<ffffffffa019ba25>] ? spl_panic+0xe5/0xf0 [spl]
[139920.298176] [<ffffffffa0193fcf>] ? spl_kmem_alloc+0xef/0x1a0 [spl]
[139920.298179] [<ffffffff810a9174>] ? __wake_up+0x34/0x50
[139920.298185] [<ffffffffa01993a0>] ? taskq_thread_create+0x100/0x100 [spl]
[139920.298191] [<ffffffffa019823c>] ? taskq_dispatch+0x6c/0x2d0 [spl]
[139920.298216] [<ffffffffa03d13e0>] ? zio_taskq_member.isra.7.constprop.10+0x80/0x80 [zfs]
[139920.298222] [<ffffffffa01984ea>] ? taskq_thread_spawn+0x4a/0x50 [spl]
[139920.298225] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139920.298246] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139920.298267] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139920.298291] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139920.298314] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139920.298321] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139920.298324] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139920.298331] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139920.298334] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139920.298337] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.298340] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139920.298343] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.298346] INFO: task z_rd_int_7:270 blocked for more than 120 seconds.
[139920.305408] Tainted: P O 3.16.0-4-amd64 #1
[139920.312485] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[139920.319651] z_rd_int_7 D ffff880035f427e8 0 270 2 0x00000000
[139920.319656] ffff880035f42390 0000000000000046 0000000000012f40 ffff88039abd3fd8
[139920.319659] 0000000000012f40 ffff880035f42390 ffffffffa0467b65 ffff88039abd3d30
[139920.319662] 00000000000014e3 ffffffffa045183f ffffffffa0468ee0 0000000000000000
[139920.319666] Call Trace:
[139920.319676] [<ffffffffa019ba25>] ? spl_panic+0xe5/0xf0 [spl]
[139920.319683] [<ffffffffa0274570>] ? fletcher_4_native+0x140/0x140 [zcommon]
[139920.319704] [<ffffffffa02d6ea8>] ? abd_iterate_func+0x118/0x3b0 [zfs]
[139920.319724] [<ffffffffa02d4250>] ? abd_copy_from_buf_off_cb+0x20/0x20 [zfs]
[139920.319744] [<ffffffffa02d783d>] ? abd_iterate_func2+0x1cd/0x720 [zfs]
[139920.319747] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139920.319768] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139920.319789] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139920.319812] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139920.319835] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139920.319842] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139920.319845] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139920.319852] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139920.319855] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139920.319858] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.319861] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139920.319864] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.319872] INFO: task txg_sync:407 blocked for more than 120 seconds.
[139920.327069] Tainted: P O 3.16.0-4-amd64 #1
[139920.334293] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[139920.341574] txg_sync D ffff880035fa0528 0 407 2 0x00000000
[139920.341578] ffff880035fa00d0 0000000000000046 0000000000012f40 ffff880399967fd8
[139920.341581] 0000000000012f40 ffff880035fa00d0 ffff8803afc937f0 ffff88036931e3c8
[139920.341584] ffff88036931e400 0000000000000001 0000000000000002 ffff8800ca72f000
[139920.341588] Call Trace:
[139920.341591] [<ffffffff81515069>] ? io_schedule+0x99/0x120
[139920.341599] [<ffffffffa019de4f>] ? cv_wait_common+0xcf/0x270 [spl]
[139920.341603] [<ffffffff810a9590>] ? prepare_to_wait_event+0xf0/0xf0
[139920.341627] [<ffffffffa03d7003>] ? zio_wait+0x203/0x3e0 [zfs]
[139920.341655] [<ffffffffa0345050>] ? dsl_scan_sync+0x7b0/0xf70 [zfs]
[139920.341685] [<ffffffffa035e898>] ? spa_sync+0x478/0x11f0 [zfs]
[139920.341713] [<ffffffffa0374d28>] ? txg_sync_thread+0x2d8/0x510 [zfs]
[139920.341742] [<ffffffffa0374a50>] ? txg_fini+0x360/0x360 [zfs]
[139920.341749] [<ffffffffa0197614>] ? thread_generic_wrapper+0x74/0xb0 [spl]
[139920.341755] [<ffffffffa01975a0>] ? __thread_exit+0x20/0x20 [spl]
[139920.341758] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139920.341762] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.341765] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139920.341768] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.341780] INFO: task z_rd_int_7:2264 blocked for more than 120 seconds.
[139920.349111] Tainted: P O 3.16.0-4-amd64 #1
[139920.356468] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[139920.363908] z_rd_int_7 D ffff880347070e78 0 2264 2 0x00000000
[139920.363915] ffff880347070a20 0000000000000046 0000000000012f40 ffff880357c1bfd8
[139920.363919] 0000000000012f40 ffff880347070a20 ffffffffa0467b65 ffff880357c1bd30
[139920.363922] 00000000000014e3 ffffffffa045183f ffffffffa0468ee0 0000000000000000
[139920.363932] Call Trace:
[139920.363944] [<ffffffffa019ba25>] ? spl_panic+0xe5/0xf0 [spl]
[139920.363951] [<ffffffffa0274570>] ? fletcher_4_native+0x140/0x140 [zcommon]
[139920.363975] [<ffffffffa02d6ea8>] ? abd_iterate_func+0x118/0x3b0 [zfs]
[139920.363980] [<ffffffff812b67aa>] ? number.isra.2+0x31a/0x350
[139920.364001] [<ffffffffa02d783d>] ? abd_iterate_func2+0x1cd/0x720 [zfs]
[139920.364012] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139920.364041] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139920.364063] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139920.364087] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139920.364111] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139920.364118] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139920.364121] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139920.364128] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139920.364131] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139920.364134] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.364137] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139920.364141] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.364144] INFO: task z_rd_int_5:2266 blocked for more than 120 seconds.
[139920.371632] Tainted: P O 3.16.0-4-amd64 #1
[139920.379167] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[139920.386795] z_rd_int_5 D ffff8803a0dab748 0 2266 2 0x00000000
[139920.386799] ffff8803a0dab2f0 0000000000000046 0000000000012f40 ffff880337c3ffd8
[139920.386802] 0000000000012f40 ffff8803a0dab2f0 ffffffffa0467b65 ffff880337c3fd30
[139920.386806] 00000000000014e3 ffffffffa045183f ffffffffa0468ee0 0000000000000000
[139920.386809] Call Trace:
[139920.386819] [<ffffffffa019ba25>] ? spl_panic+0xe5/0xf0 [spl]
[139920.386823] [<ffffffff81095ba4>] ? ttwu_do_wakeup+0x14/0xf0
[139920.386826] [<ffffffff8109818b>] ? try_to_wake_up+0x1cb/0x2f0
[139920.386829] [<ffffffff810a8f67>] ? __wake_up_common+0x57/0x90
[139920.386853] [<ffffffffa03d13e0>] ? zio_taskq_member.isra.7.constprop.10+0x80/0x80 [zfs]
[139920.386857] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139920.386878] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139920.386898] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139920.386922] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139920.386945] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139920.386952] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139920.386955] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139920.386961] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139920.386964] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139920.386967] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.386971] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139920.386974] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.386977] INFO: task z_rd_int_4:2268 blocked for more than 120 seconds.
[139920.394651] Tainted: P O 3.16.0-4-amd64 #1
[139920.402347] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[139920.410112] z_rd_int_4 D ffff88034763cef8 0 2268 2 0x00000000
[139920.410116] ffff88034763caa0 0000000000000046 0000000000012f40 ffff88039995ffd8
[139920.410120] 0000000000012f40 ffff88034763caa0 ffffffffa0467b65 ffff88039995fd30
[139920.410123] 00000000000014e3 ffffffffa045183f ffffffffa0468ee0 0000000000000000
[139920.410126] Call Trace:
[139920.410136] [<ffffffffa019ba25>] ? spl_panic+0xe5/0xf0 [spl]
[139920.410141] [<ffffffff810a1120>] ? check_preempt_wakeup+0x120/0x1d0
[139920.410144] [<ffffffff8109e9b7>] ? put_prev_entity+0x57/0x350
[139920.410147] [<ffffffff81095ba4>] ? ttwu_do_wakeup+0x14/0xf0
[139920.410150] [<ffffffff8109d7c6>] ? set_next_entity+0x56/0x70
[139920.410153] [<ffffffff810a45f1>] ? pick_next_task_fair+0x6e1/0x820
[139920.410157] [<ffffffff8101255e>] ? __switch_to+0xde/0x5a0
[139920.410161] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139920.410182] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139920.410203] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139920.410227] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139920.410250] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139920.410257] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139920.410260] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139920.410266] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139920.410269] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139920.410274] [<ffffffff810fc7d4>] ? kprobe_flush_task+0x84/0x100
[139920.410277] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.410281] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139920.410284] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.410288] INFO: task z_rd_int_3:2276 blocked for more than 120 seconds.
[139920.418147] Tainted: P O 3.16.0-4-amd64 #1
[139920.426038] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[139920.434030] z_rd_int_3 D ffff8800c9aecf78 0 2276 2 0x00000000
[139920.434034] ffff8800c9aecb20 0000000000000046 0000000000012f40 ffff88039e52ffd8
[139920.434037] 0000000000012f40 ffff8800c9aecb20 ffffffffa0467b65 ffff88039e52fd30
[139920.434040] 00000000000014e3 ffffffffa045183f ffffffffa0468ee0 0000000000000000
[139920.434044] Call Trace:
[139920.434054] [<ffffffffa019ba25>] ? spl_panic+0xe5/0xf0 [spl]
[139920.434057] [<ffffffff81095ba4>] ? ttwu_do_wakeup+0x14/0xf0
[139920.434060] [<ffffffff8109818b>] ? try_to_wake_up+0x1cb/0x2f0
[139920.434063] [<ffffffff810a8f67>] ? __wake_up_common+0x57/0x90
[139920.434065] [<ffffffff810a9174>] ? __wake_up+0x34/0x50
[139920.434090] [<ffffffffa03d13e0>] ? zio_taskq_member.isra.7.constprop.10+0x80/0x80 [zfs]
[139920.434093] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[139920.434114] [<ffffffffa02d9016>] ? buf_hash_find+0x86/0x180 [zfs]
[139920.434135] [<ffffffffa02e8154>] ? arc_read_done+0x744/0xac0 [zfs]
[139920.434158] [<ffffffffa03daf5e>] ? zio_done+0x39e/0x18b0 [zfs]
[139920.434182] [<ffffffffa03d14cd>] ? zio_execute+0xed/0x2e0 [zfs]
[139920.434188] [<ffffffffa0198cac>] ? taskq_thread+0x27c/0x580 [spl]
[139920.434192] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[139920.434198] [<ffffffffa0198a30>] ? taskq_thread_should_stop+0x90/0x90 [spl]
[139920.434201] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[139920.434204] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[139920.434207] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[139920.434210] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment