Kernel Panic on IBM Power8 w/ Nvidia Tesla P100 SXM2 with Nvidia driver 450.80.02 or 450.51.06 or 440.118.02

kernel-panic

Kernel panic - not syncing: corrupted stack end detected inside scheduler

CPU: 116 PID: 5369 Comm: lstopo Tainted: P           O      4.19.0-12-powerpc64le #1 Debian 4.19.152-1
Call Trace:
[c000001d6bba9720] [c000000000999298] dump_stack+0xbc/0x104 (unreliable)
[c000001d6bba9760] [c0000000001053d8] panic+0x140/0x318
[c000001d6bba97f0] [c0000000009b4a14] __schedule+0xac4/0xad0
[c000001d6bba98c0] [c0000000009b4e78] preempt_schedule_common+0x28/0x50
[c000001d6bba98e0] [c0000000009b4f04] _cond_resched+0x64/0x80
[c000001d6bba9910] [c00000000037e148] __kmalloc+0x2c8/0x3e0
[c000001d6bba9980] [d000000007692bc0] os_alloc_mem+0x158/0x180 [nvidia]
[c000001d6bba99b0] [d0000000080cfad8] _nv008475rm+0x48/0x70 [nvidia]
[c000001d6bba99e0] [d0000000080cf4ac] _nv008479rm+0x1c/0x40 [nvidia]
[c000001d6bba9a00] [d0000000080cf204] _nv008477rm+0x64/0x160 [nvidia]
[c000001d6bba9a30] [d000000007dfacd0] _nv007778rm+0xb0/0x550 [nvidia]
[c000001d6bba9ad0] [d000000007dfb5a8] _nv007783rm+0x218/0x3f0 [nvidia]
[c000001d6bba9c00] [d000000007df9854] _nv028549rm+0x1f4/0xbd0 [nvidia]
[c000001d6bba9d80] [d000000007dfd2d4] _nv007786rm+0x114/0x180 [nvidia]
[c000001d6bba9e00] [d000000007df98dc] _nv028549rm+0x27c/0xbd0 [nvidia]
[c000001d6bba9f80] [d000000007dfd2d4] _nv007786rm+0x114/0x180 [nvidia]
[c000001d6bbaa000] [d000000007df98dc] _nv028549rm+0x27c/0xbd0 [nvidia]
[c000001d6bbaa180] [d000000007dfd2d4] _nv007786rm+0x114/0x180 [nvidia]
[c000001d6bbaa200] [d000000007df98dc] _nv028549rm+0x27c/0xbd0 [nvidia]
[c000001d6bbaa380] [d000000007dfd2d4] _nv007786rm+0x114/0x180 [nvidia]
[c000001d6bbaa400] [d000000007dfcf64] _nv028551rm+0xf4/0x1c0 [nvidia]
[c000001d6bbaa480] [d000000007775f30] _nv023054rm+0x420/0xf70 [nvidia]
[c000001d6bbaa5f0] [d0000000080ea384] _nv004752rm+0x34/0x90 [nvidia]
[c000001d6bbaa610] [d000000007d7a03c] _nv023998rm+0x12cc/0x2600 [nvidia]
[c000001d6bbaa830] [d00000000776d77c] _nv036434rm+0x12ec/0x2e00 [nvidia]
[c000001d6bbaac00] [d0000000080e67b0] _nv003373rm+0x90/0xe0 [nvidia]
[c000001d6bbaac50] [d0000000080db54c] _nv003854rm+0x14c/0x240 [nvidia]
[c000001d6bbaace0] [d0000000077ab21c] _nv034232rm+0x5c/0xf0 [nvidia]
[c000001d6bbaad70] [d0000000080d5d08] _nv006671rm+0x228/0x4a0 [nvidia]
[c000001d6bbaaeb0] [d0000000077aa5a0] _nv035698rm+0x530/0x7c0 [nvidia]
[c000001d6bbaafe0] [d0000000080d1b2c] _nv035696rm+0x24c/0x270 [nvidia]
[c000001d6bbab080] [d0000000077aae5c] _nv034332rm+0x1cc/0x3a0 [nvidia]
[c000001d6bbab1f0] [d0000000077ab328] _nv034333rm+0x78/0xc0 [nvidia]
[c000001d6bbab330] [d0000000077aabe0] _nv034330rm+0x50/0x70 [nvidia]
[c000001d6bbab3a0] [d0000000077d7de0] _nv029250rm+0x160/0x3d0 [nvidia]
[c000001d6bbab4d0] [d0000000077d8138] _nv029283rm+0xe8/0x3e0 [nvidia]
[c000001d6bbab5c0] [d0000000077cf830] _nv029289rm+0x70/0xc0 [nvidia]
[c000001d6bbab610] [d0000000081b8a08] rm_gpu_ops_memory_alloc_sys+0x28/0x50 [nvidia]
[c000001d6bbab630] [d00000000769b13c] nvUvmInterfaceMemoryAllocSys+0x34/0x50 [nvidia]
[c000001d6bbab650] [d0000000110d1cf4] uvm_rm_mem_alloc+0xfc/0x220 [nvidia_uvm]
[c000001d6bbab730] [d0000000110d23e8] uvm_rm_mem_alloc_and_map_cpu+0x30/0xb0 [nvidia_uvm]
[c000001d6bbab770] [d000000011105e14] uvm_pushbuffer_create_common+0xac/0x210 [nvidia_uvm]
[c000001d6bbab7c0] [d0000000110d6368] uvm_channel_manager_create_common+0x320/0xc00 [nvidia_uvm]
[c000001d6bbab930] [d0000000110c48d4] uvm_gpu_retain_by_uuid+0xbac/0x1520 [nvidia_uvm]
[c000001d6bbaba70] [d0000000110cae54] uvm_va_space_register_gpu+0x4c/0x810 [nvidia_uvm]
[c000001d6bbabaf0] [d0000000110c6e00] uvm_api_register_gpu+0x48/0x70 [nvidia_uvm]
[c000001d6bbabb20] [d0000000110b578c] uvm_ioctl+0xdb4/0x15e0 [nvidia_uvm]
[c000001d6bbabc70] [d0000000110b601c] uvm_unlocked_ioctl+0x64/0xb0 [nvidia_uvm]
[c000001d6bbabcb0] [d0000000110b61f4] uvm_unlocked_ioctl_entry+0xdc/0x140 [nvidia_uvm]
[c000001d6bbabd20] [c0000000003dbe74] do_vfs_ioctl+0xd4/0x8e0
[c000001d6bbabdc0] [c0000000003dc744] ksys_ioctl+0xc4/0x110
[c000001d6bbabe10] [c0000000003dc7b8] sys_ioctl+0x28/0x80
[c000001d6bbabe30] [c00000000000b9dc] system_call+0x5c/0x70
---[ end Kernel panic - not syncing: corrupted stack end detected inside scheduler ]---