Skip to content

Instantly share code, notes, and snippets.

@pstch
Created December 14, 2016 03:36
Show Gist options
  • Save pstch/6bcad09517fe1d40f1309d9fe9bcf4a5 to your computer and use it in GitHub Desktop.
Save pstch/6bcad09517fe1d40f1309d9fe9bcf4a5 to your computer and use it in GitHub Desktop.
[117234.342190] VERIFY3(c < (1ULL << 24) >> 9) failed (36028797018963967 < 32768)
[117234.342324] PANIC at zio.c:267:zio_buf_alloc()
[117234.342414] Showing stack for process 27253
[117234.342419] CPU: 2 PID: 27253 Comm: z_wr_iss Tainted: P C O 3.16.0-4-amd64 #1 Debian 3.16.36-1+deb8u1
[117234.342421] Hardware name: Dell Inc. PowerEdge 1950/0UR033, BIOS 2.7.0 10/30/2010
[117234.342424] 0000000000000000 ffffffff81512391 ffffffffa096ebd5 ffff880323417ca0
[117234.342428] ffffffffa073438f 0000000000000000 0000000000000030 ffff880323417cb0
[117234.342431] ffff880323417c50 2833594649524556 4c553128203c2063 293432203c3c204c
[117234.342435] Call Trace:
[117234.342444] [<ffffffff81512391>] ? dump_stack+0x5d/0x78
[117234.342458] [<ffffffffa073438f>] ? spl_panic+0xbf/0xf0 [spl]
[117234.342464] [<ffffffffa0730169>] ? spl_kmem_cache_alloc+0x99/0x730 [spl]
[117234.342526] [<ffffffffa08ce951>] ? zio_buf_alloc+0x41/0x60 [zfs]
[117234.342544] [<ffffffffa08153c1>] ? abd_alloc_linear+0x61/0xe0 [zfs]
[117234.342563] [<ffffffffa08157a6>] ? abd_alloc+0x366/0x4c0 [zfs]
[117234.342582] [<ffffffffa08198c2>] ? arc_get_data_impl.isra.24+0x182/0x3a0 [zfs]
[117234.342601] [<ffffffffa0819bd5>] ? arc_hdr_alloc_abd+0xa5/0xf0 [zfs]
[117234.342619] [<ffffffffa081aa7d>] ? arc_write_ready+0x27d/0x530 [zfs]
[117234.342640] [<ffffffffa08d6bd4>] ? zio_compress_data+0x34/0xa0 [zfs]
[117234.342644] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[117234.342664] [<ffffffffa08d5ac4>] ? zio_ready+0x74/0x380 [zfs]
[117234.342686] [<ffffffffa08cfc5f>] ? zio_execute+0x8f/0xf0 [zfs]
[117234.342691] [<ffffffffa0732415>] ? taskq_thread+0x235/0x450 [spl]
[117234.342696] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[117234.342700] [<ffffffffa07321e0>] ? taskq_cancel_id+0x110/0x110 [spl]
[117234.342706] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[117234.342709] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117234.342713] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[117234.342716] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117360.224040] INFO: task z_wr_iss:27253 blocked for more than 120 seconds.
[117360.224150] Tainted: P C O 3.16.0-4-amd64 #1
[117360.224243] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[117360.224356] z_wr_iss D ffff8800b4af06a8 0 27253 2 0x00000000
[117360.224362] ffff8800b4af0250 0000000000000046 0000000000012f40 ffff880323417fd8
[117360.224365] 0000000000012f40 ffff8800b4af0250 ffffffffa096ebd5 ffff880323417ca0
[117360.224369] 000000000000010b ffffffffa094bd93 ffffffffa096eb38 ffff8803304d9630
[117360.224372] Call Trace:
[117360.224385] [<ffffffffa07343b5>] ? spl_panic+0xe5/0xf0 [spl]
[117360.224390] [<ffffffffa0730169>] ? spl_kmem_cache_alloc+0x99/0x730 [spl]
[117360.224440] [<ffffffffa08ce951>] ? zio_buf_alloc+0x41/0x60 [zfs]
[117360.224458] [<ffffffffa08153c1>] ? abd_alloc_linear+0x61/0xe0 [zfs]
[117360.224476] [<ffffffffa08157a6>] ? abd_alloc+0x366/0x4c0 [zfs]
[117360.224495] [<ffffffffa08198c2>] ? arc_get_data_impl.isra.24+0x182/0x3a0 [zfs]
[117360.224514] [<ffffffffa0819bd5>] ? arc_hdr_alloc_abd+0xa5/0xf0 [zfs]
[117360.224532] [<ffffffffa081aa7d>] ? arc_write_ready+0x27d/0x530 [zfs]
[117360.224553] [<ffffffffa08d6bd4>] ? zio_compress_data+0x34/0xa0 [zfs]
[117360.224559] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[117360.224579] [<ffffffffa08d5ac4>] ? zio_ready+0x74/0x380 [zfs]
[117360.224600] [<ffffffffa08cfc5f>] ? zio_execute+0x8f/0xf0 [zfs]
[117360.224605] [<ffffffffa0732415>] ? taskq_thread+0x235/0x450 [spl]
[117360.224610] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[117360.224615] [<ffffffffa07321e0>] ? taskq_cancel_id+0x110/0x110 [spl]
[117360.224619] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[117360.224623] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117360.224626] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[117360.224629] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117360.224637] INFO: task txg_sync:27336 blocked for more than 120 seconds.
[117360.224740] Tainted: P C O 3.16.0-4-amd64 #1
[117360.224832] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[117360.224945] txg_sync D ffff8803307cc668 0 27336 2 0x00000000
[117360.224948] ffff8803307cc210 0000000000000046 0000000000012f40 ffff8800aa9f7fd8
[117360.224951] 0000000000012f40 ffff8803307cc210 ffff8803afcd37f0 ffff8802d476ae78
[117360.224954] ffff8802d476aeb0 0000000000000001 ffff8802d476aaf8 ffff8800aa9f7d18
[117360.224958] Call Trace:
[117360.224961] [<ffffffff81515069>] ? io_schedule+0x99/0x120
[117360.224966] [<ffffffffa07360aa>] ? cv_wait_common+0x8a/0xf0 [spl]
[117360.224970] [<ffffffff810a9590>] ? prepare_to_wait_event+0xf0/0xf0
[117360.224991] [<ffffffffa08d2f6b>] ? zio_wait+0xdb/0x160 [zfs]
[117360.225017] [<ffffffffa08598aa>] ? dsl_pool_sync+0xaa/0x450 [zfs]
[117360.225045] [<ffffffffa08739a0>] ? spa_sync+0x3e0/0xd10 [zfs]
[117360.225072] [<ffffffffa0884eef>] ? txg_sync_thread+0x2ef/0x490 [zfs]
[117360.225099] [<ffffffffa0884c00>] ? txg_fini+0x2b0/0x2b0 [zfs]
[117360.225104] [<ffffffffa073136b>] ? thread_generic_wrapper+0x6b/0x80 [spl]
[117360.225108] [<ffffffffa0731300>] ? __thread_exit+0x20/0x20 [spl]
[117360.225112] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[117360.225115] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117360.225118] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[117360.225121] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117480.224037] INFO: task z_wr_iss:27253 blocked for more than 120 seconds.
[117480.224147] Tainted: P C O 3.16.0-4-amd64 #1
[117480.224239] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[117480.224352] z_wr_iss D ffff8800b4af06a8 0 27253 2 0x00000000
[117480.224357] ffff8800b4af0250 0000000000000046 0000000000012f40 ffff880323417fd8
[117480.224361] 0000000000012f40 ffff8800b4af0250 ffffffffa096ebd5 ffff880323417ca0
[117480.224364] 000000000000010b ffffffffa094bd93 ffffffffa096eb38 ffff8803304d9630
[117480.224368] Call Trace:
[117480.224381] [<ffffffffa07343b5>] ? spl_panic+0xe5/0xf0 [spl]
[117480.224386] [<ffffffffa0730169>] ? spl_kmem_cache_alloc+0x99/0x730 [spl]
[117480.224434] [<ffffffffa08ce951>] ? zio_buf_alloc+0x41/0x60 [zfs]
[117480.224452] [<ffffffffa08153c1>] ? abd_alloc_linear+0x61/0xe0 [zfs]
[117480.224471] [<ffffffffa08157a6>] ? abd_alloc+0x366/0x4c0 [zfs]
[117480.224489] [<ffffffffa08198c2>] ? arc_get_data_impl.isra.24+0x182/0x3a0 [zfs]
[117480.224508] [<ffffffffa0819bd5>] ? arc_hdr_alloc_abd+0xa5/0xf0 [zfs]
[117480.224527] [<ffffffffa081aa7d>] ? arc_write_ready+0x27d/0x530 [zfs]
[117480.224547] [<ffffffffa08d6bd4>] ? zio_compress_data+0x34/0xa0 [zfs]
[117480.224553] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[117480.224573] [<ffffffffa08d5ac4>] ? zio_ready+0x74/0x380 [zfs]
[117480.224595] [<ffffffffa08cfc5f>] ? zio_execute+0x8f/0xf0 [zfs]
[117480.224600] [<ffffffffa0732415>] ? taskq_thread+0x235/0x450 [spl]
[117480.224604] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[117480.224609] [<ffffffffa07321e0>] ? taskq_cancel_id+0x110/0x110 [spl]
[117480.224613] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[117480.224616] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117480.224620] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[117480.224623] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117480.224631] INFO: task txg_sync:27336 blocked for more than 120 seconds.
[117480.224734] Tainted: P C O 3.16.0-4-amd64 #1
[117480.224826] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[117480.224940] txg_sync D ffff8803307cc668 0 27336 2 0x00000000
[117480.224943] ffff8803307cc210 0000000000000046 0000000000012f40 ffff8800aa9f7fd8
[117480.224946] 0000000000012f40 ffff8803307cc210 ffff8803afcd37f0 ffff8802d476ae78
[117480.224949] ffff8802d476aeb0 0000000000000001 ffff8802d476aaf8 ffff8800aa9f7d18
[117480.224953] Call Trace:
[117480.224956] [<ffffffff81515069>] ? io_schedule+0x99/0x120
[117480.224961] [<ffffffffa07360aa>] ? cv_wait_common+0x8a/0xf0 [spl]
[117480.224965] [<ffffffff810a9590>] ? prepare_to_wait_event+0xf0/0xf0
[117480.224986] [<ffffffffa08d2f6b>] ? zio_wait+0xdb/0x160 [zfs]
[117480.225012] [<ffffffffa08598aa>] ? dsl_pool_sync+0xaa/0x450 [zfs]
[117480.225040] [<ffffffffa08739a0>] ? spa_sync+0x3e0/0xd10 [zfs]
[117480.225067] [<ffffffffa0884eef>] ? txg_sync_thread+0x2ef/0x490 [zfs]
[117480.225094] [<ffffffffa0884c00>] ? txg_fini+0x2b0/0x2b0 [zfs]
[117480.225099] [<ffffffffa073136b>] ? thread_generic_wrapper+0x6b/0x80 [spl]
[117480.225103] [<ffffffffa0731300>] ? __thread_exit+0x20/0x20 [spl]
[117480.225107] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[117480.225110] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117480.225113] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[117480.225116] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117480.225122] INFO: task useradd:20844 blocked for more than 120 seconds.
[117480.225222] Tainted: P C O 3.16.0-4-amd64 #1
[117480.225314] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[117480.225427] useradd D ffff8802beb0e4a8 0 20844 20839 0x00000000
[117480.225430] ffff8802beb0e050 0000000000000086 0000000000012f40 ffff88008fdaffd8
[117480.225433] 0000000000012f40 ffff8802beb0e050 ffff88034dfc42c0 ffff88034dfc4210
[117480.225436] ffff88034dfc42c8 0000000000000000 ffff88034dfc4278 ffff88034e1b1a00
[117480.225439] Call Trace:
[117480.225445] [<ffffffffa07360dd>] ? cv_wait_common+0xbd/0xf0 [spl]
[117480.225448] [<ffffffff810a9590>] ? prepare_to_wait_event+0xf0/0xf0
[117480.225475] [<ffffffffa0884377>] ? txg_wait_synced+0xd7/0x110 [zfs]
[117480.225497] [<ffffffffa08cbd3c>] ? zil_create+0x20c/0x2b0 [zfs]
[117480.225519] [<ffffffffa08ccc13>] ? zil_commit.part.11+0x723/0x750 [zfs]
[117480.225522] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[117480.225544] [<ffffffffa08c2d33>] ? zfs_fsync+0x73/0xf0 [zfs]
[117480.225564] [<ffffffffa08db15f>] ? zpl_fsync+0x5f/0x90 [zfs]
[117480.225569] [<ffffffff811d82eb>] ? do_fsync+0x4b/0x70
[117480.225573] [<ffffffff811d855c>] ? SyS_fsync+0xc/0x10
[117480.225576] [<ffffffff8151854d>] ? system_call_fast_compare_end+0x10/0x15
[117600.224027] INFO: task z_wr_iss:27253 blocked for more than 120 seconds.
[117600.224136] Tainted: P C O 3.16.0-4-amd64 #1
[117600.224228] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[117600.224341] z_wr_iss D ffff8800b4af06a8 0 27253 2 0x00000000
[117600.224346] ffff8800b4af0250 0000000000000046 0000000000012f40 ffff880323417fd8
[117600.224349] 0000000000012f40 ffff8800b4af0250 ffffffffa096ebd5 ffff880323417ca0
[117600.224352] 000000000000010b ffffffffa094bd93 ffffffffa096eb38 ffff8803304d9630
[117600.224356] Call Trace:
[117600.224367] [<ffffffffa07343b5>] ? spl_panic+0xe5/0xf0 [spl]
[117600.224373] [<ffffffffa0730169>] ? spl_kmem_cache_alloc+0x99/0x730 [spl]
[117600.224412] [<ffffffffa08ce951>] ? zio_buf_alloc+0x41/0x60 [zfs]
[117600.224430] [<ffffffffa08153c1>] ? abd_alloc_linear+0x61/0xe0 [zfs]
[117600.224448] [<ffffffffa08157a6>] ? abd_alloc+0x366/0x4c0 [zfs]
[117600.224467] [<ffffffffa08198c2>] ? arc_get_data_impl.isra.24+0x182/0x3a0 [zfs]
[117600.224486] [<ffffffffa0819bd5>] ? arc_hdr_alloc_abd+0xa5/0xf0 [zfs]
[117600.224505] [<ffffffffa081aa7d>] ? arc_write_ready+0x27d/0x530 [zfs]
[117600.224525] [<ffffffffa08d6bd4>] ? zio_compress_data+0x34/0xa0 [zfs]
[117600.224529] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[117600.224550] [<ffffffffa08d5ac4>] ? zio_ready+0x74/0x380 [zfs]
[117600.224571] [<ffffffffa08cfc5f>] ? zio_execute+0x8f/0xf0 [zfs]
[117600.224576] [<ffffffffa0732415>] ? taskq_thread+0x235/0x450 [spl]
[117600.224580] [<ffffffff810982e0>] ? wake_up_state+0x10/0x10
[117600.224585] [<ffffffffa07321e0>] ? taskq_cancel_id+0x110/0x110 [spl]
[117600.224589] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[117600.224592] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117600.224596] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[117600.224599] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117600.224604] INFO: task txg_sync:27336 blocked for more than 120 seconds.
[117600.224708] Tainted: P C O 3.16.0-4-amd64 #1
[117600.224799] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[117600.224912] txg_sync D ffff8803307cc668 0 27336 2 0x00000000
[117600.224916] ffff8803307cc210 0000000000000046 0000000000012f40 ffff8800aa9f7fd8
[117600.224919] 0000000000012f40 ffff8803307cc210 ffff8803afcd37f0 ffff8802d476ae78
[117600.224922] ffff8802d476aeb0 0000000000000001 ffff8802d476aaf8 ffff8800aa9f7d18
[117600.224925] Call Trace:
[117600.224929] [<ffffffff81515069>] ? io_schedule+0x99/0x120
[117600.224934] [<ffffffffa07360aa>] ? cv_wait_common+0x8a/0xf0 [spl]
[117600.224938] [<ffffffff810a9590>] ? prepare_to_wait_event+0xf0/0xf0
[117600.224959] [<ffffffffa08d2f6b>] ? zio_wait+0xdb/0x160 [zfs]
[117600.224985] [<ffffffffa08598aa>] ? dsl_pool_sync+0xaa/0x450 [zfs]
[117600.225012] [<ffffffffa08739a0>] ? spa_sync+0x3e0/0xd10 [zfs]
[117600.225040] [<ffffffffa0884eef>] ? txg_sync_thread+0x2ef/0x490 [zfs]
[117600.225067] [<ffffffffa0884c00>] ? txg_fini+0x2b0/0x2b0 [zfs]
[117600.225072] [<ffffffffa073136b>] ? thread_generic_wrapper+0x6b/0x80 [spl]
[117600.225076] [<ffffffffa0731300>] ? __thread_exit+0x20/0x20 [spl]
[117600.225079] [<ffffffff810894bd>] ? kthread+0xbd/0xe0
[117600.225083] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117600.225086] [<ffffffff81518498>] ? ret_from_fork+0x58/0x90
[117600.225089] [<ffffffff81089400>] ? kthread_create_on_node+0x180/0x180
[117600.225093] INFO: task useradd:20844 blocked for more than 120 seconds.
[117600.225193] Tainted: P C O 3.16.0-4-amd64 #1
[117600.225284] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[117600.225397] useradd D ffff8802beb0e4a8 0 20844 20839 0x00000000
[117600.225400] ffff8802beb0e050 0000000000000086 0000000000012f40 ffff88008fdaffd8
[117600.225404] 0000000000012f40 ffff8802beb0e050 ffff88034dfc42c0 ffff88034dfc4210
[117600.225407] ffff88034dfc42c8 0000000000000000 ffff88034dfc4278 ffff88034e1b1a00
[117600.225410] Call Trace:
[117600.225415] [<ffffffffa07360dd>] ? cv_wait_common+0xbd/0xf0 [spl]
[117600.225418] [<ffffffff810a9590>] ? prepare_to_wait_event+0xf0/0xf0
[117600.225445] [<ffffffffa0884377>] ? txg_wait_synced+0xd7/0x110 [zfs]
[117600.225467] [<ffffffffa08cbd3c>] ? zil_create+0x20c/0x2b0 [zfs]
[117600.225489] [<ffffffffa08ccc13>] ? zil_commit.part.11+0x723/0x750 [zfs]
[117600.225492] [<ffffffff81516d0e>] ? mutex_lock+0xe/0x2a
[117600.225514] [<ffffffffa08c2d33>] ? zfs_fsync+0x73/0xf0 [zfs]
[117600.225534] [<ffffffffa08db15f>] ? zpl_fsync+0x5f/0x90 [zfs]
[117600.225539] [<ffffffff811d82eb>] ? do_fsync+0x4b/0x70
[117600.225542] [<ffffffff811d855c>] ? SyS_fsync+0xc/0x10
[117600.225546] [<ffffffff8151854d>] ? system_call_fast_compare_end+0x10/0x15
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment