Skip to content

Instantly share code, notes, and snippets.

@b10n1k

b10n1k/txt

Created July 20, 2022 07:34
Show Gist options
  • Save b10n1k/4cbe68ea3f0d07057b2651d86c570e64 to your computer and use it in GitHub Desktop.
Save b10n1k/4cbe68ea3f0d07057b2651d86c570e64 to your computer and use it in GitHub Desktop.
Download ZIP
lttng start trace
Raw
txt
[ 255.759835] trampoline signature fail
[ 255.759855] ------------[ cut here ]------------
[ 255.759857] kernel BUG at ../arch/x86/kernel/static_call.c:76!
[ 255.759871] invalid opcode: 0000 [#1] PREEMPT_RT SMP PTI
[ 255.759879] CPU: 0 PID: 2507 Comm: lttng-sessiond Tainted: G O X 5.14.21-150400.10-rt #1 SLE15-SP4 1d02e8875b7fea8f4379b86fda0f4392217c8600
[ 255.759885] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.14.0-0-g155821a-rebuilt.opensuse.org 04/01/2014
[ 255.759887] RIP: 0010:__static_call_validate+0x25/0x90
[ 255.759898] Code: cc cc cc cc cc 0f 1f 44 00 00 84 d2 0f b6 07 74 27 0f b7 0d b9 d4 fb 00 66 39 4f 05 74 0e 48 c7 c7 fe 06 73 a8 e8 a9 4a 9e 00 <0f> 0b 0f b6 0d a0 d4 fb 00 38 4f 07 75 e6 40 84 f6 75 2d 3c e8 74
[ 255.759900] RSP: 0018:ffffa6350087b998 EFLAGS: 00010282
[ 255.759904] RAX: 0000000000000019 RBX: 0000000000000000 RCX: 0000000000000000
[ 255.759906] RDX: 0000000000000000 RSI: ffff9b247dc229c0 RDI: ffff9b247dc229c0
[ 255.759908] RBP: ffffffffc1732000 R08: ffffffffa8e84d00 R09: 74616e6769732065
[ 255.759909] R10: ffffa6350087b9d0 R11: 0000000000000019 R12: 0000000000000000
[ 255.759910] R13: ffffffffc12b46b8 R14: ffffffffc1732000 R15: ffffffffc12b7030
[ 255.759912] FS: 00007fa8867fc700(0000) GS:ffff9b247dc00000(0000) knlGS:0000000000000000
[ 255.759915] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 255.759916] CR2: 00007fc15fff4f30 CR3: 0000000047b22000 CR4: 00000000000006f0
[ 255.759921] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 255.759922] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 255.759924] Call Trace:
[ 255.759936] <TASK>
[ 255.759940] arch_static_call_transform+0x39/0x90
[ 255.759948] ? __SCT__tp_func_lttng_statedump_network_interface+0x8/0x8 [lttng_statedump 4c2bcd4fabbfbb1ffad3a58a2cbe73ca54539497]
[ 255.759956] __static_call_update+0x65/0x240
[ 255.759969] ? 0xffffffffc1732000
[ 255.759971] ? __kmalloc+0x136/0x2d0
[ 255.759978] ? tracepoint_add_func+0xd7/0x300
[ 255.759987] tracepoint_add_func+0x253/0x300
[ 255.759992] ? 0xffffffffc1732000
[ 255.759994] tracepoint_probe_register_prio+0x55/0x90
[ 255.759999] ? 0xffffffffc1732000
[ 255.760001] ? 0xffffffffc1732000
[ 255.760002] lttng_tracepoint_probe_register+0xc0/0x110 [lttng_tracer 01d81dc567556102817bf418781bd723650404ed]
[ 255.760081] lttng_session_sync_enablers+0x657/0x680 [lttng_tracer 01d81dc567556102817bf418781bd723650404ed]
[ 255.760126] lttng_session_enable+0x35/0x110 [lttng_tracer 01d81dc567556102817bf418781bd723650404ed]
[ 255.760168] lttng_session_ioctl+0x2e8/0x670 [lttng_tracer 01d81dc567556102817bf418781bd723650404ed]
[ 255.760210] ? wait_woken+0x70/0x70
[ 255.760215] ? unix_stream_recvmsg+0x53/0x80
[ 255.760222] ? unix_set_peek_off+0x50/0x50
[ 255.760226] ? ____sys_recvmsg+0xf7/0x190
[ 255.760233] ? import_iovec+0x17/0x20
[ 255.760237] ? copy_msghdr_from_user+0x5c/0x90
[ 255.760242] ? ___sys_recvmsg+0x9e/0xe0
[ 255.760248] __x64_sys_ioctl+0x92/0xd0
[ 255.760258] do_syscall_64+0x5b/0x80
[ 255.760273] ? exit_to_user_mode_prepare+0x1ef/0x260
[ 255.760279] ? syscall_exit_to_user_mode+0x18/0x40
[ 255.760283] ? do_syscall_64+0x67/0x80
[ 255.760286] ? exit_to_user_mode_prepare+0x1ef/0x260
[ 255.760289] ? syscall_exit_to_user_mode+0x18/0x40
[ 255.760292] ? do_syscall_64+0x67/0x80
[ 255.760295] ? exc_page_fault+0x6a/0x190
[ 255.760299] entry_SYSCALL_64_after_hwframe+0x61/0xcb
[ 255.760309] RIP: 0033:0x7fa88e98cc47
[ 255.760313] Code: 90 90 90 48 8b 05 49 c2 2d 00 64 c7 00 26 00 00 00 48 c7 c0 ff ff ff ff c3 66 2e 0f 1f 84 00 00 00 00 00 b8 10 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 19 c2 2d 00 f7 d8 64 89 01 48
[ 255.760315] RSP: 002b:00007fa8867e4488 EFLAGS: 00000246 ORIG_RAX: 0000000000000010
[ 255.760318] RAX: ffffffffffffffda RBX: 0000000000000042 RCX: 00007fa88e98cc47
[ 255.760320] RDX: 0000000100000000 RSI: 000000000000f656 RDI: 0000000000000042
[ 255.760322] RBP: 00007fa8740072e0 R08: 000055f4a70fe288 R09: 0000000000000000
[ 255.760323] R10: 0000000000000001 R11: 0000000000000246 R12: 0000000000000000
[ 255.760324] R13: fffffffffffffe80 R14: 00007fa8867e45a0 R15: 0000000000000000
[ 255.760327] </TASK>
[ 255.760328] Modules linked in: lttng_probe_x86_exceptions(O) lttng_probe_x86_irq_vectors(O) lttng_probe_writeback(O) lttng_probe_workqueue(O) lttng_probe_v4l2(O) lttng_probe_vmscan(O) lttng_probe_udp(O) lttng_probe_timer(O) lttng_probe_sunrpc(O) lttng_probe_statedump(O) lttng_probe_sock(O) lttng_probe_skb(O) lttng_probe_signal(O) lttng_probe_scsi(O) lttng_probe_sched(O) ltt\
ng_probe_regulator(O) lttng_probe_rcu(O) lttng_probe_random(O) lttng_probe_printk(O) lttng_probe_power(O) lttng_probe_net(O) lttng_probe_napi(O) lttng_probe_module(O) kvm irqbypass lttng_probe_kvm(O) lttng_probe_kmem(O) lttng_probe_jbd2(O) lttng_probe_irq(O) lttng_probe_i2c(O) lttng_probe_gpio(O) lttng_probe_compaction(O) lttng_probe_block(O) lttng_probe_asoc(O) lttng_ring_buf\
fer_metadata_mmap_client(O) lttng_ring_buffer_client_mmap_overwrite(O) lttng_ring_buffer_client_mmap_discard(O) lttng_ring_buffer_metadata_client(O) lttng_ring_buffer_client_overwrite(O) lttng_ring_buffer_client_discard(O)
[ 255.760365] lttng_tracer(O) lttng_statedump(O) lttng_wrapper(O) lttng_kprobes(O) lttng_clock(O) lttng_uprobes(O) lttng_lib_ring_buffer(O) lttng_kretprobes(O) md_cluster md_mod oracleasm(OX) crash(O) reiserfs gfs2 ocfs2_stack_user dlm ocfs2_stack_o2cb ocfs2 jbd2 quota_tree ocfs2_dlmfs ocfs2_stackglue ocfs2_dlm ocfs2_nodemanager af_packet nft_fib_inet nft_fib_ipv4 nft_fib_ip\
v6 nft_fib nft_reject_inet nf_reject_ipv4 nf_reject_ipv6 nft_reject nft_ct nft_chain_nat nf_tables ebtable_nat ebtable_broute ip6table_nat ip6table_mangle ip6table_raw ip6table_security iptable_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 iptable_mangle iptable_raw iptable_security ip_set bridge stp nfnetlink llc iscsi_ibft iscsi_boot_sysfs ebtable_filter ebtables rfk\
ill ip6table_filter ip6_tables iptable_filter bpfilter snd_hda_codec_generic ledtrig_audio snd_hda_intel snd_intel_dspcfg snd_intel_sdw_acpi snd_hda_codec ppdev hid_generic snd_hda_core snd_hwdep snd_pcm usbhid snd_timer pcspkr
[ 255.760448] joydev virtio_net snd net_failover parport_pc failover soundcore parport i2c_piix4 button xfs fuse configfs ip_tables x_tables bochs_drm drm_vram_helper drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops cec xhci_pci rc_core ata_generic xhci_pci_renesas drm_ttm_helper ttm sr_mod cdrom xhci_hcd ata_piix ahci libahci drm usbcore libata serio_raw virtio_\
blk virtio_scsi floppy qemu_fw_cfg btrfs blake2b_generic libcrc32c xor raid6_pq sg dm_multipath dm_mod scsi_dh_rdac scsi_dh_emc scsi_dh_alua scsi_mod msr virtio_rng
[ 255.760543] Supported: Yes, External
[ 255.856893] ---[ end trace aa25ff4626a0ad95 ]---
[ 255.856895] RIP: 0010:__static_call_validate+0x25/0x90
[ 255.856900] Code: cc cc cc cc cc 0f 1f 44 00 00 84 d2 0f b6 07 74 27 0f b7 0d b9 d4 fb 00 66 39 4f 05 74 0e 48 c7 c7 fe 06 73 a8 e8 a9 4a 9e 00 <0f> 0b 0f b6 0d a0 d4 fb 00 38 4f 07 75 e6 40 84 f6 75 2d 3c e8 74
[ 255.856901] RSP: 0018:ffffa6350087b998 EFLAGS: 00010282
[ 255.856903] RAX: 0000000000000019 RBX: 0000000000000000 RCX: 0000000000000000
[ 255.856904] RDX: 0000000000000000 RSI: ffff9b247dc229c0 RDI: ffff9b247dc229c0
[ 255.856904] RBP: ffffffffc1732000 R08: ffffffffa8e84d00 R09: 74616e6769732065
[ 255.856905] R10: ffffa6350087b9d0 R11: 0000000000000019 R12: 0000000000000000
[ 255.856906] R13: ffffffffc12b46b8 R14: ffffffffc1732000 R15: ffffffffc12b7030
[ 255.856906] FS: 00007fa8867fc700(0000) GS:ffff9b247dc00000(0000) knlGS:0000000000000000
[ 255.856907] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 255.856908] CR2: 00007f777002a648 CR3: 0000000047b22000 CR4: 00000000000006f0
[ 255.856913] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 255.856914] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
@jonathanmills
Copy link

I am seeing this exact same error with other sles 15 SP4 kernels and Mellanox NICs.....it's driving me crazy

@jonathanmills
Copy link

[ 18.878051] ib_core: externally supported module, setting X kernel taint flag.
[ 18.895206] ib_uverbs: externally supported module, setting X kernel taint flag.
[ 20.267725] mlx5_core 0000:5e:00.0: probe_one:2136:(pid 934): mlx5_init_one failed with error code -22
[ 20.286960] mlx5_core: probe of 0000:5e:00.0 failed with error -22
[ 20.296823] mlx5_core 0000:d8:00.0: firmware version: 16.35.2000
[ 20.305048] mlx5_core 0000:d8:00.0: 63.008 Gb/s available PCIe bandwidth (8.0 GT/s PCIe x8 link)
[ 20.779931] mlx5_core 0000:d8:00.0: Rate limit: 127 rates are supported, range: 0Mbps to 24414Mbps
[ 20.799601] mlx5_core 0000:d8:00.0: E-Switch: Total vports 10, per vport: max uc(128) max mc(2048)
[ 20.823688] mlx5_core 0000:d8:00.0: Port module event: module 0, Cable plugged
[ 20.833290] mlx5_core 0000:d8:00.0: mlx5_pcie_event:306:(pid 8): PCIe slot advertised sufficient power (27W).
[ 20.860002] mlx5_core 0000:d8:00.0: mlx5_fw_tracer_start:821:(pid 415): FWTracer: Ownership granted and active
[ 20.907829] mlx5_core 0000:d8:00.0: E-Switch: Disable: mode(LEGACY), nvfs(0), active vports(0)
[ 22.340637] mlx5_core 0000:d8:00.0: E-Switch: cleanup
[ 23.174810] mlx5_core 0000:d8:00.0: probe_one:2136:(pid 415): mlx5_init_one failed with error code -22
[ 23.194140] mlx5_core: probe of 0000:d8:00.0 failed with error -22
[ 23.203688] mlx5_core 0000:d8:00.1: firmware version: 16.35.2000
[ 23.211012] mlx5_core 0000:d8:00.1: 63.008 Gb/s available PCIe bandwidth (8.0 GT/s PCIe x8 link)
[ 23.682324] mlx5_core 0000:d8:00.1: Rate limit: 127 rates are supported, range: 0Mbps to 24414Mbps
[ 23.701914] mlx5_core 0000:d8:00.1: E-Switch: Total vports 10, per vport: max uc(128) max mc(2048)
[ 23.725955] mlx5_core 0000:d8:00.1: Port module event: module 1, Cable plugged
[ 23.735889] mlx5_core 0000:d8:00.1: mlx5_pcie_event:306:(pid 936): PCIe slot advertised sufficient power (27W).
[ 23.763547] mlx5_core 0000:d8:00.1: mlx5_fw_tracer_start:821:(pid 415): FWTracer: Ownership granted and active
[ 23.803830] mlx5_core 0000:d8:00.1: E-Switch: Disable: mode(LEGACY), nvfs(0), active vports(0)
[ 25.172501] mlx5_core 0000:d8:00.1: E-Switch: cleanup
[ 26.030638] mlx5_core 0000:d8:00.1: probe_one:2136:(pid 415): mlx5_init_one failed with error code -22
[ 26.050098] mlx5_core: probe of 0000:d8:00.1 failed with error -22
[ 26.069908] mlx5_ib: externally supported module, setting X kernel taint flag.
[ 26.170835] trampoline signature fail
[ 26.170842] ------------[ cut here ]------------
[ 26.184811] kernel BUG at ../arch/x86/kernel/static_call.c:76!
[ 26.192400] invalid opcode: 0000 [#1] PREEMPT SMP NOPTI
[ 26.199720] CPU: 4 PID: 1636 Comm: openibd Kdump: loaded Tainted: G OE X 5.14.21-150400.24.46-default #1 SLE15-SP4 98cc77d94566d5eead15db9029d52a2ca42a9eb7
[ 26.217193] Hardware name: Supermicro SYS-1029P-WTR/X11DDW-L, BIOS 3.8a 10/28/2022
[ 26.225940] RIP: 0010:__static_call_validate+0x25/0x90
[ 26.232311] Code: cc cc cc cc cc 0f 1f 44 00 00 84 d2 0f b6 07 74 27 0f b7 0d 59 f5 fb 00 66 39 4f 05 74 0e 48 c7 c7 fe 5a 13 96 e8 33 9b 9e 00 <0f> 0b 0f b6 0d 40 f5 fb 00 38 4f 07 75 e6 40 84 f6 75 2d 3c e8 74
[ 26.252817] RSP: 0018:ffffadc7cf657ca0 EFLAGS: 00010286
[ 26.259236] RAX: 0000000000000019 RBX: 0000000000000000 RCX: 0000000000000000
[ 26.267569] RDX: 0000000000000000 RSI: ffff8b1f80d22940 RDI: ffff8b1f80d22940
[ 26.275920] RBP: ffffffffc10ffb70 R08: 0000000000000000 R09: 0000000000000001
[ 26.284292] R10: 000000000000002f R11: ffffadc7cf657ac8 R12: 0000000000000000
[ 26.292652] R13: ffffffffc11bfb50 R14: ffffffffc10ffb70 R15: ffffffffc126f790
[ 26.301014] FS: 00007f3d886c62c0(0000) GS:ffff8b1f80d00000(0000) knlGS:0000000000000000
[ 26.310299] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 26.317269] CR2: 000055b6dbbb69f0 CR3: 0000000184590004 CR4: 00000000007706e0
[ 26.325609] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 26.333939] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 26.342261] PKRU: 55555554
[ 26.346152] Call Trace:
[ 26.349774]
[ 26.353091] arch_static_call_transform+0x39/0x90
[ 26.358964] ? __SCT__tp_func_mlx5_fs_del_rule+0x8/0x8 [mlx5_core 1567037a23fe2d7a9c3b46279a9a42df9ccec3c0]
[ 26.369946] __static_call_update+0x65/0x240
[ 26.375395] ? mlx5_fw_tracer_set_core_dump_reg.constprop.21+0x90/0x90 [mlx5_core 1567037a23fe2d7a9c3b46279a9a42df9ccec3c0]
[ 26.388289] ? text_poke_finish+0x1b/0x30
[ 26.393488] tracepoint_add_func+0x253/0x300
[ 26.398935] ? mlx5_fw_tracer_set_core_dump_reg.constprop.21+0x90/0x90 [mlx5_core 1567037a23fe2d7a9c3b46279a9a42df9ccec3c0]
[ 26.411770] tracepoint_probe_register_prio+0x55/0x90
[ 26.418020] ? mlx5_fw_tracer_set_core_dump_reg.constprop.21+0x90/0x90 [mlx5_core 1567037a23fe2d7a9c3b46279a9a42df9ccec3c0]
[ 26.430875] __ftrace_event_enable_disable+0x9a/0x280
[ 26.437152] event_enable_write+0x92/0xf0
[ 26.442349] vfs_write+0xb9/0x280
[ 26.446849] ksys_write+0xa5/0xe0
[ 26.451349] do_syscall_64+0x58/0x80
[ 26.456131] ? syscall_exit_to_user_mode+0x18/0x40
[ 26.462087] ? do_syscall_64+0x67/0x80
[ 26.466987] ? syscall_exit_to_user_mode+0x18/0x40
[ 26.472974] ? do_syscall_64+0x67/0x80
[ 26.477876] ? do_syscall_64+0x67/0x80
[ 26.482763] ? exc_page_fault+0x67/0x150
[ 26.487836] entry_SYSCALL_64_after_hwframe+0x61/0xcb
[ 26.494015] RIP: 0033:0x7f3d87d7baf3
[ 26.498716] Code: 0f 1f 80 00 00 00 00 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 64 8b 04 25 18 00 00 00 85 c0 75 14 b8 01 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 55 f3 c3 0f 1f 00 41 54 55 49 89 d4 53 48 89
[ 26.519062] RSP: 002b:00007ffdd2dfe448 EFLAGS: 00000246 ORIG_RAX: 0000000000000001
[ 26.527773] RAX: ffffffffffffffda RBX: 0000000000000002 RCX: 00007f3d87d7baf3
[ 26.536087] RDX: 0000000000000002 RSI: 000055b6dbbc8de0 RDI: 0000000000000001
[ 26.544379] RBP: 000055b6dbbc8de0 R08: 000000000000000a R09: 0000000000000000
[ 26.552659] R10: 000055b6dbcad150 R11: 0000000000000246 R12: 00007f3d8805f500
[ 26.560932] R13: 0000000000000002 R14: 00007f3d88064c00 R15: 0000000000000002
[ 26.569193]
[ 26.572502] Modules linked in: mlx5_ib(OEX) ib_uverbs(OEX) ib_core(OEX) rfkill dmi_sysfs ipmi_ssif intel_rapl_msr intel_rapl_common skx_edac nfit libnvdimm mlx5_core(OEX) mlxdevm(OEX) mlx_compat(OEX) iTCO_wdt x86_pkg_temp_thermal psample intel_pmc_bxt acpi_ipmi intel_powerclamp iTCO_vendor_support mlxfw(OEX) ipmi_si ioatdma mei_me ipmi_devintf lpc_ich tls i2c_i801 joydev pcspkr i40e pci_hyperv_intf(X) mfd_core i2c_smbus mei intel_pch_thermal coretemp dca ipmi_msghandler acpi_pad button fuse configfs ip_tables x_tables xfs libcrc32c hid_generic usbhid ast i2c_algo_bit drm_vram_helper sd_mod drm_kms_helper crc32_pclmul t10_pi crc32c_intel syscopyarea sysfillrect xhci_pci sysimgblt xhci_pci_renesas fb_sys_fops ghash_clmulni_intel xhci_hcd cec aesni_intel rc_core drm_ttm_helper ttm crypto_simd drm cryptd ahci usbcore libahci megaraid_sas libata dm_mirror dm_region_hash dm_log sg dm_multipath dm_mod scsi_dh_rdac scsi_dh_emc scsi_dh_alua scsi_mod msr
[ 26.660463] Supported: Yes, External

@Aenerine
Copy link

I have ancountered the same error on OpenSUSE LEAP 15.4 running on ESXI. Kernel version: 5.14.21-150400.24.46-default.
Its really not nice to start a kernel debugger / tracer and realize that it is itselft buggy.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment