From 76b59936dab9134fbf0b81b1de7960b88ab970ef Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Thu, 22 Nov 2018 07:10:16 +0100 Subject: pkg/report: add "PANIC: double fault" report format Amusing that's another kernel failure mode that we are discovering after 3 years. One can't even reliably understand when kernel has crashed. I wonder if syzkaller never hit these, or just never recognized and reported them. We will see. Don't even want to think about arm kernel output parsing. --- pkg/report/testdata/linux/report/103 | 2 +- pkg/report/testdata/linux/report/313 | 50 ++++++++++++++++++++++++++++++++++++ pkg/report/testdata/linux/report/314 | 43 +++++++++++++++++++++++++++++++ pkg/report/testdata/linux/report/315 | 46 +++++++++++++++++++++++++++++++++ pkg/report/testdata/linux/report/316 | 24 +++++++++++++++++ 5 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 pkg/report/testdata/linux/report/313 create mode 100644 pkg/report/testdata/linux/report/314 create mode 100644 pkg/report/testdata/linux/report/315 create mode 100644 pkg/report/testdata/linux/report/316 (limited to 'pkg/report/testdata/linux') diff --git a/pkg/report/testdata/linux/report/103 b/pkg/report/testdata/linux/report/103 index 727ff8dfa..f06a7c54d 100644 --- a/pkg/report/testdata/linux/report/103 +++ b/pkg/report/testdata/linux/report/103 @@ -1,4 +1,4 @@ -TITLE: divide error: 0000 [#1] SMP KASAN +TITLE: divide error in corrupted CORRUPTED: Y [ 89.659427] netlink: 13 bytes leftover after parsing attributes in process syz-executor5'. diff --git a/pkg/report/testdata/linux/report/313 b/pkg/report/testdata/linux/report/313 new file mode 100644 index 000000000..cbe2d063e --- /dev/null +++ b/pkg/report/testdata/linux/report/313 @@ -0,0 +1,50 @@ +TITLE: PANIC: double fault in kvm_async_pf_task_wait + +[ 4031.436692] PANIC: double fault, error_code: 0x0 +[ 4031.439937] CPU: 1 PID: 1227 Comm: kworker/1:1 Not tainted 4.16.0-rc1+ #12 +[ 4031.440632] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-2.fc27 04/01/2014 +[ 4031.441475] Workqueue: events netstamp_clear +[ 4031.441897] RIP: 0010:kvm_async_pf_task_wait+0x19d/0x250 +[ 4031.442411] RSP: 0000:ffffc90000f5fbc0 EFLAGS: 00000202 +[ 4031.442916] RAX: ffff880136048000 RBX: ffffc90000f5fbe0 RCX: 0000000000000006 +[ 4031.443601] RDX: 0000000000000006 RSI: ffff880136048a40 RDI: ffff880136048000 +[ 4031.444285] RBP: ffffc90000f5fc90 R08: 000005212156f5cf R09: 0000000000000000 +[ 4031.444966] R10: 0000000000000000 R11: 0000000000000000 R12: ffffc90000f5fbf0 +[ 4031.445650] R13: 0000000000002e88 R14: ffffffff82ad6360 R15: 0000000000000000 +[ 4031.446335] FS: 0000000000000000(0000) GS:ffff88013a800000(0000) knlGS:0000000000000000 +[ 4031.447104] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 4031.447659] CR2: 0000000000006001 CR3: 0000000002212006 CR4: 00000000001606e0 +[ 4031.448354] Call Trace: +[ 4031.448602] ? kvm_clock_read+0x1f/0x30 +[ 4031.448985] ? prepare_to_swait+0x1d/0x70 +[ 4031.449384] ? trace_hardirqs_off_thunk+0x1a/0x1c +[ 4031.449845] ? do_async_page_fault+0x67/0x90 +[ 4031.450283] do_async_page_fault+0x67/0x90 +[ 4031.450684] async_page_fault+0x25/0x50 +[ 4031.451067] RIP: 0010:text_poke+0x60/0x250 +[ 4031.451528] RSP: 0000:ffffc90000f5fd78 EFLAGS: 00010286 +[ 4031.452082] RAX: ffffea0000000000 RBX: ffffea000005f200 RCX: ffffffff817c88e9 +[ 4031.452843] RDX: 0000000000000001 RSI: ffffc90000f5fdbf RDI: ffffffff817c88e4 +[ 4031.453568] RBP: ffffffff817c88e4 R08: 0000000000000000 R09: 0000000000000001 +[ 4031.454283] R10: ffffc90000f5fdf0 R11: 104eab8665f42bc7 R12: 0000000000000001 +[ 4031.455010] R13: ffffc90000f5fdbf R14: ffffffff817c98e4 R15: 0000000000000000 +[ 4031.455719] ? dev_gro_receive+0x3f4/0x6f0 +[ 4031.456123] ? netif_receive_skb_internal+0x24/0x380 +[ 4031.456641] ? netif_receive_skb_internal+0x29/0x380 +[ 4031.457202] ? netif_receive_skb_internal+0x24/0x380 +[ 4031.457743] ? text_poke+0x28/0x250 +[ 4031.458084] ? netif_receive_skb_internal+0x24/0x380 +[ 4031.458567] ? netif_receive_skb_internal+0x25/0x380 +[ 4031.459046] text_poke_bp+0x55/0xe0 +[ 4031.459393] arch_jump_label_transform+0x90/0xf0 +[ 4031.459842] __jump_label_update+0x63/0x70 +[ 4031.460243] static_key_enable_cpuslocked+0x54/0x80 +[ 4031.460713] static_key_enable+0x16/0x20 +[ 4031.461096] process_one_work+0x266/0x6d0 +[ 4031.461506] worker_thread+0x3a/0x390 +[ 4031.462328] ? process_one_work+0x6d0/0x6d0 +[ 4031.463299] kthread+0x121/0x140 +[ 4031.464122] ? kthread_create_worker_on_cpu+0x70/0x70 +[ 4031.465070] ret_from_fork+0x3a/0x50 +[ 4031.465859] Code: 89 58 08 4c 89 f7 49 89 9d 20 35 ad 82 48 89 95 58 ff ff ff 4c 8d 63 10 e8 61 e4 8d 00 eb 22 e8 7a d7 0b 00 fb 66 0f 1f 44 00 00 be 64 8d 00 fa 66 0f 1f 44 00 00 e8 12 a0 0b 00 e8 cd 7a 0e +[ 4031.468817] Kernel panic - not syncing: Machine halted. diff --git a/pkg/report/testdata/linux/report/314 b/pkg/report/testdata/linux/report/314 new file mode 100644 index 000000000..db8bff2c9 --- /dev/null +++ b/pkg/report/testdata/linux/report/314 @@ -0,0 +1,43 @@ +# This is not corrupted but we don't understand this stack trace format. +TITLE: PANIC: double fault in compat_sock_ioctl +CORRUPTED: Y + +[ 10.584905] PANIC: double fault, error_code: 0x0 +[ 10.585372] CPU: 0 PID: 917 Comm: netifd Not tainted 4.14.0-rc1-00015-gf5caf62 #2 +[ 10.586062] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.9.3-20161025_171302-gandalf 04/01/2014 +[ 10.586978] task: ffff8800141a8000 task.stack: ffffc90000274000 +[ 10.587519] RIP: 0010:compat_sock_ioctl+0xfea/0x103e +[ 10.587974] RSP: 0000:0000000000277d78 EFLAGS: 00010283 +[ 10.588448] RAX: 0000000000277d78 RBX: 0000000000008933 RCX: ffff8800141a8000 +[ 10.589103] RDX: 0000000000000020 RSI: 00000000fffbea00 RDI: 00000000fffbea50 +[ 10.589757] RBP: ffffc90000277e18 R08: fffbea50fffbea34 R09: ffffffff814a68c9 +[ 10.590407] R10: ffffff9c00000002 R11: 00000000fffbea50 R12: 0000000000000000 +[ 10.591056] R13: ffff880012c8c880 R14: 00000000fffbea50 R15: 00000000fffbea00 +[ 10.591708] FS: 0000000000000000(0000) GS:ffff880019a00000(0063) knlGS:00000000f7fab9a0 +[ 10.592446] CS: 0010 DS: 002b ES: 002b CR0: 0000000080050033 +[ 10.592973] CR2: 0000000000277d68 CR3: 000000001807f000 CR4: 00000000000006b0 +[ 10.593623] Call Trace: +[ 10.593858] Code: 02 0f ff 65 48 8b 04 25 80 d1 00 00 48 8b 80 28 25 00 00 48 83 e8 20 49 39 c7 77 34 89 e0 4c 89 f7 4c 89 fe ba 20 00 00 00 89 c4 b3 52 05 00 85 c0 74 22 eb 1a 4c 89 fa 89 de 4c 89 ef e8 c6 +[ 10.595705] Kernel panic - not syncing: Machine halted. +[ 10.596181] CPU: 0 PID: 917 Comm: netifd Not tainted 4.14.0-rc1-00015-gf5caf62 #2 +[ 10.596868] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.9.3-20161025_171302-gandalf 04/01/2014 +[ 10.597786] Call Trace: +[ 10.598017] <#DF> +[ 10.598216] dump_stack+0x8f/0xc4 +[ 10.598527] panic+0xda/0x22f +[ 10.598811] df_debug+0x30/0x30 +[ 10.599104] do_double_fault+0x9b/0xaf +[ 10.599456] double_fault+0x22/0x30 +[ 10.599791] RIP: 0010:compat_sock_ioctl+0xfea/0x103e +[ 10.600247] RSP: 0000:0000000000277d78 EFLAGS: 00010283 +[ 10.600733] RAX: 0000000000277d78 RBX: 0000000000008933 RCX: ffff8800141a8000 +[ 10.601381] RDX: 0000000000000020 RSI: 00000000fffbea00 RDI: 00000000fffbea50 +[ 10.602036] RBP: ffffc90000277e18 R08: fffbea50fffbea34 R09: ffffffff814a68c9 +[ 10.602700] R10: ffffff9c00000002 R11: 00000000fffbea50 R12: 0000000000000000 +[ 10.603349] R13: ffff880012c8c880 R14: 00000000fffbea50 R15: 00000000fffbea00 +[ 10.604002] ? dev_ioctl+0x2d7/0x5e3 +[ 10.604336] WARNING: kernel stack regs at ffff880019a05f58 in netifd:917 has bad 'bp' value ffffc90000277e18 +[ 10.604337] unwind stack type:0 next_sp:0000000000277d78 mask:0x10 graph_idx:0 +[ 10.604339] ffff880019a05d48: ffff880019a05e28 (0xffff880019a05e28) +[ 10.604340] ffff880019a05d50: ffffffff8100ce27 (show_trace_log_lvl+0x203/0x2a7) +[ 10.604342] ffff880019a05d58: ffffffff81473d28 (compat_sock_ioctl+0xfea/0x103e) diff --git a/pkg/report/testdata/linux/report/315 b/pkg/report/testdata/linux/report/315 new file mode 100644 index 000000000..461b6c8c3 --- /dev/null +++ b/pkg/report/testdata/linux/report/315 @@ -0,0 +1,46 @@ +# This is not corrupted but we don't understand this stack trace format. +TITLE: PANIC: double fault in trace_hardirqs_off_thunk +CORRUPTED: Y + +[ 0.004000] PANIC: double fault, error_code: 0x0 +[ 0.004000] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.18.0-rc4-00074-g19efe000 #2 +[ 0.004000] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1 04/01/2014 +[ 0.004000] RIP: 0010:trace_hardirqs_off_thunk+0xb/0x1c +[ 0.004000] Code: 5f 5d c3 55 48 89 e5 57 56 52 51 50 41 50 41 51 41 52 41 53 48 8b 7d 08 e8 f5 4e 2b 00 eb 34 55 48 89 e5 57 56 52 51 50 41 50 <41> 51 41 52 41 53 48 8b 7d 08 e8 eb 52 2b 00 eb 18 55 48 89 e5 57 +[ 0.004000] RSP: 0000:ffffc90000000000 EFLAGS: 00010087 +[ 0.004000] RAX: 0000000082800a97 RBX: 0000000000000001 RCX: ffffffff82800a97 +[ 0.004000] RDX: 0000000000000000 RSI: ffffffff82800f68 RDI: ffffffff83678c68 +[ 0.004000] RBP: ffffc90000000030 R08: 0000000000000000 R09: 0000000000000000 +[ 0.004000] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000 +[ 0.004000] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000 +[ 0.004000] FS: 0000000000000000(0000) GS:ffff880013400000(0000) knlGS:0000000000000000 +[ 0.004000] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 0.004000] CR2: ffffc8fffffffff8 CR3: 000000000366e000 CR4: 00000000000406b0 +[ 0.004000] Call Trace: +[ 0.004000] Kernel panic - not syncing: Machine halted. +[ 0.004000] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.18.0-rc4-00074-g19efe000 #2 +[ 0.004000] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1 04/01/2014 +[ 0.004000] Call Trace: +[ 0.004000] <#DF> +[ 0.004000] dump_stack+0x9b/0xe7 +[ 0.004000] panic+0x1ad/0x325 +[ 0.004000] ? refcount_error_report+0x267/0x267 +[ 0.004000] df_debug+0x32/0x32 +[ 0.004000] do_double_fault+0x1b3/0x1c7 +[ 0.004000] double_fault+0x23/0x30 +[ 0.004000] RIP: 0010:trace_hardirqs_off_thunk+0xb/0x1c +[ 0.004000] Code: 5f 5d c3 55 48 89 e5 57 56 52 51 50 41 50 41 51 41 52 41 53 48 8b 7d 08 e8 f5 4e 2b 00 eb 34 55 48 89 e5 57 56 52 51 50 41 50 <41> 51 41 52 41 53 48 8b 7d 08 e8 eb 52 2b 00 eb 18 55 48 89 e5 57 +[ 0.004000] RSP: 0000:ffffc90000000000 EFLAGS: 00010087 +[ 0.004000] RAX: 0000000082800a97 RBX: 0000000000000001 RCX: ffffffff82800a97 +[ 0.004000] RDX: 0000000000000000 RSI: ffffffff82800f68 RDI: ffffffff83678c68 +[ 0.004000] RBP: ffffc90000000030 R08: 0000000000000000 R09: 0000000000000000 +[ 0.004000] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000 +[ 0.004000] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000 +[ 0.004000] ? native_iret+0x7/0x7 +[ 0.004000] ? async_page_fault+0x8/0x30 +[ 0.004000] WARNING: kernel stack regs at (____ptrval____) in swapper/0:0 has bad 'bp' value (____ptrval____) +[ 0.004000] unwind stack type:0 next_sp:(____ptrval____) mask:0x20 graph_idx:0 +[ 0.004000] (____ptrval____): fffffe0000008e00 (0xfffffe0000008e00) +[ 0.004000] (____ptrval____): ffffffff81045a52 (show_trace_log_lvl+0x1ec/0x2c8) +[ 0.004000] (____ptrval____): ffffffff82800f68 (async_page_fault+0x8/0x30) +[ 0.004000] (____ptrval____): fffffe0000008fd8 (0xfffffe0000008fd8) diff --git a/pkg/report/testdata/linux/report/316 b/pkg/report/testdata/linux/report/316 new file mode 100644 index 000000000..4c7822c8d --- /dev/null +++ b/pkg/report/testdata/linux/report/316 @@ -0,0 +1,24 @@ +TITLE: kernel panic: corrupted stack end detected inside scheduler + +[ 2231.649459] Kernel panic - not syncing: corrupted stack end detected inside scheduler +[ 2231.657307] CPU: 185 PID: 11718 Comm: dbus-daemon Kdump: loaded Tainted: G W 4.20.0-rc2+ #4 +[ 2231.676788] Call trace: +[ 2231.679273] dump_backtrace+0x0/0x2c8 +[ 2231.682950] show_stack+0x24/0x30 +[ 2231.686273] dump_stack+0x118/0x19c +[ 2231.689765] panic+0x1b8/0x31c +[ 2231.692822] schedule+0x0/0x240 +[ 2231.695963] preempt_schedule_common+0x3c/0x78 +[ 2231.700406] _cond_resched+0xfc/0x108 +[ 2231.704077] kmem_cache_alloc+0x2e0/0x3f8 +[ 2231.708102] selinux_inode_alloc_security+0xc4/0x1b0 +[ 2231.713080] security_inode_alloc+0x44/0x70 +[ 2231.717267] inode_init_always+0x270/0x4b8 +[ 2231.721364] alloc_inode+0x50/0xd0 +[ 2231.724768] new_inode_pseudo+0x84/0x120 +[ 2231.728691] sock_alloc+0x30/0x108 +[ 2231.732093] __sock_create+0x154/0x560 +[ 2231.735843] __sys_socket+0xc8/0x178 +[ 2231.739429] __arm64_sys_socket+0x4c/0x60 +[ 2231.743460] el0_svc_handler+0xd4/0x198 +[ 2231.747295] el0_svc+0x8/0xc -- cgit mrf-deployment