Skip to content

Instantly share code, notes, and snippets.

@nrdmn

nrdmn/crash.c

Last active September 5, 2019 08:01
Show Gist options
  • Save nrdmn/c2fd8b1a09e3c3c3598d2991f5470a70 to your computer and use it in GitHub Desktop.
Save nrdmn/c2fd8b1a09e3c3c3598d2991f5470a70 to your computer and use it in GitHub Desktop.
Download ZIP
atop bug
Raw
crash.c
#include <linux/perf_event.h>
#include <linux/hw_breakpoint.h>
#include <string.h>
#include <unistd.h>
#include <asm/unistd.h>
#include <stdio.h>
int main()
{
if (geteuid() != 0) {
fputs("program must be run as root!\n", stderr);
return 1;
}
struct perf_event_attr foo;
memset(&foo, 0, sizeof(struct perf_event_attr));
foo.type = PERF_TYPE_HARDWARE;
foo.size = sizeof(struct perf_event_attr);
foo.inherit = 1;
foo.pinned = 1;
foo.config = PERF_COUNT_HW_INSTRUCTIONS;
syscall(__NR_perf_event_open, &foo, -1, 0, -1, PERF_FLAG_FD_CLOEXEC);
return 0;
}
@nrdmn
Copy link
Author

nrdmn commented Sep 3, 2019 

[   15.101373] general protection fault: 0000 [#1] SMP 
[   15.102206] Modules linked in: ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype br_netfilter ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) ppdev kvm_amd kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev virtio_rng pcspkr virtio_balloon parport_pc parport sg i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom virtio_console ata_generic pata_acpi virtio_blk virtio_net qxl drm_kms_helper syscopyarea sysfillrect crct10dif_pclmul sysimgblt serio_raw crct10dif_common fb_sys_fops ttm crc32c_intel ata_piix drm libata floppy nvme nvme_core virtio_pci virtio_ring virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
[   15.102206] CPU: 0 PID: 6074 Comm: crash Kdump: loaded Tainted: G               ------------ T 3.10.0-957.12.2.el7.x86_64 #1
[   15.102206] Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
[   15.102206] task: ffff9ec3e3f9e180 ti: ffff9ec37bcb8000 task.ti: ffff9ec37bcb8000
[   15.102206] RIP: 0010:[<ffffffff89c6a376>]  [<ffffffff89c6a376>] native_read_pmc+0x6/0x20
[   15.102206] RSP: 0018:ffff9ec37bcbbad0  EFLAGS: 00010083
[   15.102206] RAX: 0000000000000001 RBX: ffff9ec3ffc0e280 RCX: 0000000000000000
[   15.102206] RDX: 0000000000000000 RSI: 00000000001300c0 RDI: 0000000000000000
[   15.102206] RBP: ffff9ec37bcbbad0 R08: ffff9ec380e0c438 R09: 0000000000000000
[   15.102206] R10: ffff9ec3eb05b530 R11: ffff9ec383498310 R12: ffff800000000001
[   15.145275] R13: ffff9ec380e0c000 R14: ffff9ec380e0c180 R15: 0000000000000010
[   15.145275] FS:  00007f0ed62ce740(0000) GS:ffff9ec3ffc00000(0000) knlGS:0000000000000000
[   15.145275] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   15.145275] CR2: 00007f0ed5d68bb0 CR3: 0000000092210000 CR4: 00000000003407f0
[   15.145275] Call Trace:
[   15.145275]  [<ffffffff89c03d1b>] x86_perf_event_update+0x4b/0xa0
[   15.145275]  [<ffffffff89c03dc8>] x86_pmu_stop+0x58/0xd0
[   15.145275]  [<ffffffff89c03e9a>] x86_pmu_del+0x5a/0x160
[   15.145275]  [<ffffffff89da5258>] event_sched_out.isra.94+0x78/0x200
[   15.145275]  [<ffffffff89da541e>] __perf_remove_from_context+0x3e/0x130
[   15.145275]  [<ffffffff89d9e8a8>] event_function+0xa8/0x180
[   15.145275]  [<ffffffff89da53e0>] ? event_sched_out.isra.94+0x200/0x200
[   15.145275]  [<ffffffff89da05ca>] remote_function+0x4a/0x50
[   15.145275]  [<ffffffff89d113d1>] generic_exec_single+0x161/0x1b0
[   15.145275]  [<ffffffff89da0580>] ? perf_cgroup_attach+0x60/0x60
[   15.145275]  [<ffffffff89d1147f>] smp_call_function_single+0x5f/0xa0
[   15.145275]  [<ffffffffc04cdd62>] ? xfs_iext_lookup_extent+0x52/0x70 [xfs]
[   15.145275]  [<ffffffff89d9f753>] cpu_function_call+0x43/0x60
[   15.145275]  [<ffffffff89d9e800>] ? retprobe_show+0x30/0x30
[   15.145275]  [<ffffffff89da4461>] event_function_call+0x101/0x110
[   15.145275]  [<ffffffff89da53e0>] ? event_sched_out.isra.94+0x200/0x200
[   15.145275]  [<ffffffff89da4675>] perf_remove_from_context+0x25/0x90
[   15.145275]  [<ffffffff89da8439>] perf_event_release_kernel+0xe9/0x300
[   15.145275]  [<ffffffff89da8660>] perf_release+0x10/0x20
[   15.145275]  [<ffffffff89e43b4c>] __fput+0xec/0x260
[   15.145275]  [<ffffffff89e43dae>] ____fput+0xe/0x10
[   15.145275]  [<ffffffff89cbe88b>] task_work_run+0xbb/0xe0
[   15.145275]  [<ffffffff89c9dd51>] do_exit+0x2d1/0xa40
[   15.145275]  [<ffffffff89c9e53f>] do_group_exit+0x3f/0xa0
[   15.145275]  [<ffffffff89c9e5b4>] SyS_exit_group+0x14/0x20
[   15.145275]  [<ffffffff8a375ddb>] system_call_fastpath+0x22/0x27
[   15.145275] Code: c0 48 c1 e2 20 89 0e 48 09 c2 48 89 d0 5d c3 66 0f 1f 44 00 00 55 89 f0 89 f9 48 89 e5 0f 30 31 c0 5d c3 66 90 55 89 f9 48 89 e5 <0f> 33 89 c0 48 c1 e2 20 48 09 c2 48 89 d0 5d c3 66 2e 0f 1f 84 
[   15.145275] RIP  [<ffffffff89c6a376>] native_read_pmc+0x6/0x20
[   15.145275]  RSP <ffff9ec37bcbbad0>

@nrdmn
Copy link
Author

nrdmn commented Sep 3, 2019 

[   24.846816] general protection fault: 0000 [#1] SMP 
[   24.847438] Modules linked in: ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype br_netfilter ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) ppdev kvm_amd kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr virtio_rng virtio_balloon sg parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom virtio_console virtio_net virtio_blk ata_generic pata_acpi qxl drm_kms_helper crct10dif_pclmul crct10dif_common crc32c_intel serio_raw syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nvme ata_piix floppy libata nvme_core virtio_pci virtio_ring virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
[   24.847438] CPU: 0 PID: 6451 Comm: crash Kdump: loaded Tainted: G               ------------ T 3.10.0-957.12.2.el7.x86_64 #1
[   24.847438] Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
[   24.847438] task: ffff89f45564b0c0 ti: ffff89f43b9fc000 task.ti: ffff89f43b9fc000
[   24.847438] RIP: 0010:[<ffffffff8fc6a376>]  [<ffffffff8fc6a376>] native_read_pmc+0x6/0x20
[   24.847438] RSP: 0018:ffff89f43b9ffad0  EFLAGS: 00010083
[   24.847438] RAX: 0000000000000001 RBX: ffff89f4ffc0e280 RCX: 0000000000000000
[   24.847438] RDX: 0000000000000000 RSI: 00000000001300c0 RDI: 0000000000000000
[   24.847438] RBP: ffff89f43b9ffad0 R08: ffff89f4c0ec8438 R09: 0000000000000000
[   24.847438] R10: ffff89f4ff05b530 R11: ffff89f484b55710 R12: ffff800000000001
[   24.847438] R13: ffff89f4c0ec8000 R14: ffff89f4c0ec8180 R15: 0000000000000010
[   24.847438] FS:  00007f0af5582740(0000) GS:ffff89f4ffc00000(0000) knlGS:0000000000000000
[   24.847438] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   24.847438] CR2: 00007f0af501cbb0 CR3: 000000053d010000 CR4: 00000000003407f0
[   24.847438] Call Trace:
[   24.847438]  [<ffffffff8fc03d1b>] x86_perf_event_update+0x4b/0xa0
[   24.847438]  [<ffffffff8fc03dc8>] x86_pmu_stop+0x58/0xd0
[   24.847438]  [<ffffffff8fc03e9a>] x86_pmu_del+0x5a/0x160
[   24.847438]  [<ffffffff8fda5258>] event_sched_out.isra.94+0x78/0x200
[   24.847438]  [<ffffffff8fda541e>] __perf_remove_from_context+0x3e/0x130
[   24.847438]  [<ffffffff8fd9e8a8>] event_function+0xa8/0x180
[   24.847438]  [<ffffffff8fda53e0>] ? event_sched_out.isra.94+0x200/0x200
[   24.847438]  [<ffffffff8fda05ca>] remote_function+0x4a/0x50
[   24.847438]  [<ffffffff8fd113d1>] generic_exec_single+0x161/0x1b0
[   24.847438]  [<ffffffff8fda0580>] ? perf_cgroup_attach+0x60/0x60
[   24.847438]  [<ffffffff8fd1147f>] smp_call_function_single+0x5f/0xa0
[   24.847438]  [<ffffffffc0574d62>] ? xfs_iext_lookup_extent+0x52/0x70 [xfs]
[   24.847438]  [<ffffffff8fd9f753>] cpu_function_call+0x43/0x60
[   24.847438]  [<ffffffff8fd9e800>] ? retprobe_show+0x30/0x30
[   24.847438]  [<ffffffff8fda4461>] event_function_call+0x101/0x110
[   24.847438]  [<ffffffff8fda53e0>] ? event_sched_out.isra.94+0x200/0x200
[   24.847438]  [<ffffffff8fda4675>] perf_remove_from_context+0x25/0x90
[   24.847438]  [<ffffffff8fda8439>] perf_event_release_kernel+0xe9/0x300
[   24.847438]  [<ffffffff8fda8660>] perf_release+0x10/0x20
[   24.847438]  [<ffffffff8fe43b4c>] __fput+0xec/0x260
[   24.847438]  [<ffffffff8fe43dae>] ____fput+0xe/0x10
[   24.847438]  [<ffffffff8fcbe88b>] task_work_run+0xbb/0xe0
[   24.847438]  [<ffffffff8fc9dd51>] do_exit+0x2d1/0xa40
[   24.847438]  [<ffffffff8fc9e53f>] do_group_exit+0x3f/0xa0
[   24.847438]  [<ffffffff8fc9e5b4>] SyS_exit_group+0x14/0x20
[   24.847438]  [<ffffffff90375ddb>] system_call_fastpath+0x22/0x27
[   24.847438] Code: c0 48 c1 e2 20 89 0e 48 09 c2 48 89 d0 5d c3 66 0f 1f 44 00 00 55 89 f0 89 f9 48 89 e5 0f 30 31 c0 5d c3 66 90 55 89 f9 48 89 e5 <0f> 33 89 c0 48 c1 e2 20 48 09 c2 48 89 d0 5d c3 66 2e 0f 1f 84 
[   24.847438] RIP  [<ffffffff8fc6a376>] native_read_pmc+0x6/0x20
[   24.847438]  RSP <ffff89f43b9ffad0>

@nrdmn
Copy link
Author

nrdmn commented Sep 3, 2019

# gdb /usr/lib/debug/usr/lib/modules/3.10.0-957.27.2.el7.x86_64/vmlinux

(gdb) list *native_read_pmc
0xffffffff810191b0 is in native_read_pmc (arch/x86/include/asm/msr.h:165).
160	
161	/* Deprecated, keep it for a cycle for easier merging: */
162	#define rdtscll(now)	do { (now) = rdtsc_ordered(); } while (0)
163	
164	static inline unsigned long long native_read_pmc(int counter)
165	{
166		DECLARE_ARGS(val, low, high);
167	
168		asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter));
169		return EAX_EDX_VAL(val, low, high);

(gdb) disas /r *native_read_pmc
Dump of assembler code for function native_read_pmc:
   0xffffffff810191b0 <+0>:	55	push   %rbp
   0xffffffff810191b1 <+1>:	89 f9	mov    %edi,%ecx
   0xffffffff810191b3 <+3>:	48 89 e5	mov    %rsp,%rbp
   0xffffffff810191b6 <+6>:	0f 33	rdpmc  
   0xffffffff810191b8 <+8>:	89 c0	mov    %eax,%eax
   0xffffffff810191ba <+10>:	48 c1 e2 20	shl    $0x20,%rdx
   0xffffffff810191be <+14>:	48 09 c2	or     %rax,%rdx
   0xffffffff810191c1 <+17>:	48 89 d0	mov    %rdx,%rax
   0xffffffff810191c4 <+20>:	5d	pop    %rbp
   0xffffffff810191c5 <+21>:	c3	retq   
End of assembler dump.

@nrdmn
Copy link
Author

nrdmn commented Sep 3, 2019 

PCE = 1<<8
CR4.PCE == 1

@nrdmn
Copy link
Author

nrdmn commented Sep 3, 2019 

[   19.149966] general protection fault: 0000 [#1] SMP 
[   19.150866] Modules linked in: ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype br_netfilter ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) ppdev kvm_amd kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr parport_pc virtio_rng parport sg virtio_balloon i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic virtio_blk virtio_net virtio_console pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel qxl drm_kms_helper floppy syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix serio_raw drm libata nvme nvme_core virtio_pci virtio_ring virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
[   19.152195] CPU: 0 PID: 6403 Comm: bstore_kv_final Tainted: G               ------------ T 3.10.0-957.27.2.el7.x86_64 #1
[   19.152195] Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
[   19.152195] task: ffff9c07997c30c0 ti: ffff9c078de90000 task.ti: ffff9c078de90000
[   19.152195] RIP: 0010:[<ffffffffad06b456>]  [<ffffffffad06b456>] native_read_pmc+0x6/0x20
[   19.152195] RSP: 0000:ffff9c07bfc03e50  EFLAGS: 00010083
[   19.152195] RAX: 0000000000000001 RBX: ffff9c07bfc0e280 RCX: 0000000000000000
[   19.152195] RDX: 0000000000000000 RSI: 00000000001300c0 RDI: 0000000000000000
[   19.152195] RBP: ffff9c07bfc03e50 R08: ffff9c0799bcdc38 R09: 0000000000000092
[   19.152195] R10: 0000000000000000 R11: 0000000000000000 R12: ffff800000000001
[   19.152195] R13: ffff9c0799bcd800 R14: ffff9c0799bcd980 R15: 0000000000000010
[   19.152195] FS:  00007fd9269cb700(0000) GS:ffff9c07bfc00000(0000) knlGS:0000000000000000
[   19.152195] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   19.152195] CR2: 00000000006dee20 CR3: 0000000517648000 CR4: 00000000003407f0
[   19.152195] Call Trace:
[   19.152195]  <IRQ> 
[   19.152195]  [<ffffffffad003d1b>] x86_perf_event_update+0x4b/0xa0
[   19.152195]  [<ffffffffad003dc8>] x86_pmu_stop+0x58/0xd0
[   19.152195]  [<ffffffffad003e9a>] x86_pmu_del+0x5a/0x160
[   19.152195]  [<ffffffffad1a63a8>] event_sched_out.isra.94+0x78/0x200
[   19.152195]  [<ffffffffad1a656e>] __perf_remove_from_context+0x3e/0x130
[   19.152195]  [<ffffffffad19f9f8>] event_function+0xa8/0x180
[   19.152195]  [<ffffffffad1a171a>] remote_function+0x4a/0x50
[   19.152195]  [<ffffffffad1127a3>] flush_smp_call_function_queue+0x63/0x130
[   19.152195]  [<ffffffffad112ea3>] generic_smp_call_function_single_interrupt+0x13/0x30
[   19.152195]  [<ffffffffad05747d>] smp_call_function_single_interrupt+0x2d/0x40
[   19.152195]  [<ffffffffad7790a2>] call_function_single_interrupt+0x162/0x170
[   19.152195]  <EOI> 
[   19.152195] Code: c0 48 c1 e2 20 89 0e 48 09 c2 48 89 d0 5d c3 66 0f 1f 44 00 00 55 89 f0 89 f9 48 89 e5 0f 30 31 c0 5d c3 66 90 55 89 f9 48 89 e5 <0f> 33 89 c0 48 c1 e2 20 48 09 c2 48 89 d0 5d c3 66 2e 0f 1f 84 
[   19.152195] RIP  [<ffffffffad06b456>] native_read_pmc+0x6/0x20
[   19.152195]  RSP <ffff9c07bfc03e50>
[   19.152195] ---[ end trace e1f85674715c61b4 ]---
[   19.152195] Kernel panic - not syncing: Fatal exception in interrupt
[   19.152195] Kernel Offset: 0x2c000000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff)

@nrdmn
Copy link
Author

nrdmn commented Sep 5, 2019

Fixed in 5.2

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment