From e419f4e6d851d09202e550ac2a5975ef68fd6935 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Tue, 7 Jul 2020 09:35:45 +0200 Subject: pkg/report: consider on_each_cpu as stall anchor function Kernel frequently hangs in on_each_cpu, it seems to be more indicative of the root cause than subsequent frames. --- pkg/report/testdata/linux/report/297 | 2 +- pkg/report/testdata/linux/report/300 | 2 +- pkg/report/testdata/linux/report/350 | 2 +- pkg/report/testdata/linux/report/502 | 64 ++++++++++++++ pkg/report/testdata/linux/report/503 | 141 ++++++++++++++++++++++++++++++ pkg/report/testdata/linux/report/504 | 163 +++++++++++++++++++++++++++++++++++ pkg/report/testdata/linux/report/505 | 151 ++++++++++++++++++++++++++++++++ pkg/report/testdata/linux/report/65 | 2 +- 8 files changed, 523 insertions(+), 4 deletions(-) create mode 100644 pkg/report/testdata/linux/report/502 create mode 100644 pkg/report/testdata/linux/report/503 create mode 100644 pkg/report/testdata/linux/report/504 create mode 100644 pkg/report/testdata/linux/report/505 (limited to 'pkg/report/testdata/linux') diff --git a/pkg/report/testdata/linux/report/297 b/pkg/report/testdata/linux/report/297 index 0d126f9a4..32465775c 100644 --- a/pkg/report/testdata/linux/report/297 +++ b/pkg/report/testdata/linux/report/297 @@ -1,4 +1,4 @@ -TITLE: BUG: soft lockup in blkdev_close +TITLE: BUG: soft lockup in on_each_cpu TYPE: HANG [ 601.962573] watchdog: BUG: soft lockup - CPU#1 stuck for 123s! [syz-executor1:24793] diff --git a/pkg/report/testdata/linux/report/300 b/pkg/report/testdata/linux/report/300 index bb0d0b02f..449e65f4d 100644 --- a/pkg/report/testdata/linux/report/300 +++ b/pkg/report/testdata/linux/report/300 @@ -1,4 +1,4 @@ -TITLE: INFO: rcu detected stall in uhaul_release +TITLE: INFO: rcu detected stall in on_each_cpu TYPE: HANG [ 3481.239960] INFO: rcu_sched detected stalls on CPUs/tasks: diff --git a/pkg/report/testdata/linux/report/350 b/pkg/report/testdata/linux/report/350 index 7c2eb3ce1..bce97503f 100644 --- a/pkg/report/testdata/linux/report/350 +++ b/pkg/report/testdata/linux/report/350 @@ -1,4 +1,4 @@ -TITLE: BUG: soft lockup in perf_release +TITLE: BUG: soft lockup in on_each_cpu TYPE: HANG [ 1098.601132][ C2] watchdog: BUG: soft lockup - CPU#2 stuck for 136s! [syz-executor4:4888] diff --git a/pkg/report/testdata/linux/report/502 b/pkg/report/testdata/linux/report/502 new file mode 100644 index 000000000..404e8674a --- /dev/null +++ b/pkg/report/testdata/linux/report/502 @@ -0,0 +1,64 @@ +TITLE: INFO: rcu detected stall in on_each_cpu +TYPE: HANG + +[ 611.778481][ C1] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 611.785841][ C1] rcu: Tasks blocked on level-0 rcu_node (CPUs 0-1): +[ 611.785848][ C1] ------------[ cut here ]------------ +[ 611.798299][ C1] IRQs not enabled as expected +[ 611.803253][ C1] WARNING: CPU: 1 PID: 4737 at kernel/sched/core.c:2698 try_invoke_on_locked_down_task+0x18b/0x320 +[ 611.815146][ C1] Kernel panic - not syncing: panic_on_warn set ... +[ 611.822004][ C1] CPU: 1 PID: 4737 Comm: systemd-timesyn Not tainted 5.8.0-rc3-syzkaller #0 +[ 611.830678][ C1] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 611.840748][ C1] Call Trace: +[ 611.844041][ C1] +[ 611.847038][ C1] dump_stack+0x1f0/0x31e +[ 611.851537][ C1] panic+0x264/0x7a0 +[ 611.872226][ C1] __warn+0x227/0x250 +[ 611.882552][ C1] report_bug+0x1b1/0x2e0 +[ 611.887028][ C1] handle_bug+0x42/0x80 +[ 611.891294][ C1] exc_invalid_op+0x16/0x40 +[ 611.895859][ C1] asm_exc_invalid_op+0x12/0x20 +[ 611.900723][ C1] RIP: 0010:try_invoke_on_locked_down_task+0x18b/0x320 +[ 611.907579][ C1] Code: 48 89 df e8 a7 35 09 00 4c 89 f7 e8 3f 63 ce 06 e9 b5 00 00 00 c6 05 92 7d 18 08 01 48 c7 c7 fc a9 e7 88 31 c0 e8 65 a8 f5 ff <0f> 0b e9 15 ff ff ff 48 c7 c1 70 5d 6d 89 80 e1 07 80 c1 03 38 c1 +[ 611.927186][ C1] RSP: 0018:ffffc90000da8c50 EFLAGS: 00010046 +[ 611.933365][ C1] RAX: 82cfb6c0e4d32500 RBX: ffff888096c1ee08 RCX: ffff888096c1e580 +[ 611.941446][ C1] RDX: 0000000000010004 RSI: 0000000000010004 RDI: 0000000000000000 +[ 611.949435][ C1] RBP: ffff888096c1ee4c R08: ffffffff815dc419 R09: ffffed1015d241c3 +[ 611.957429][ C1] R10: ffffed1015d241c3 R11: 0000000000000000 R12: 0000000000000000 +[ 611.965404][ C1] R13: ffff8880a9990040 R14: ffff8880a99903c0 R15: dffffc0000000000 +[ 611.983534][ C1] rcu_sched_clock_irq+0x12ec/0x1eb0 +[ 611.993836][ C1] update_process_times+0x12c/0x180 +[ 611.999126][ C1] tick_sched_timer+0x254/0x410 +[ 612.009559][ C1] __hrtimer_run_queues+0x42d/0x930 +[ 612.014781][ C1] hrtimer_interrupt+0x373/0xd60 +[ 612.019801][ C1] __sysvec_apic_timer_interrupt+0xf0/0x260 +[ 612.025733][ C1] asm_call_on_stack+0xf/0x20 +[ 612.030407][ C1] +[ 612.033350][ C1] sysvec_apic_timer_interrupt+0xb9/0x130 +[ 612.039081][ C1] asm_sysvec_apic_timer_interrupt+0x12/0x20 +[ 612.045068][ C1] RIP: 0010:smp_call_function_single+0x311/0x450 +[ 612.051394][ C1] Code: 83 e5 01 75 14 e8 4f 00 0b 00 eb 45 0f 1f 00 66 2e 0f 1f 84 00 00 00 00 00 f3 90 42 0f b6 04 23 84 c0 75 0d 41 f6 07 01 74 22 2a 00 0b 00 eb e8 44 89 f9 80 e1 07 80 c1 03 38 c1 7c e6 4c 89 +[ 612.071000][ C1] RSP: 0018:ffffc90001c27be0 EFLAGS: 00000202 +[ 612.077068][ C1] RAX: 0000000000000000 RBX: 1ffff92000384f81 RCX: ffff888096c1e580 +[ 612.085045][ C1] RDX: 0000000000000000 RSI: 0000000000000001 RDI: 0000000000000000 +[ 612.093027][ C1] RBP: ffffc90001c27c80 R08: ffffffff81697bb6 R09: ffffed1015d06d11 +[ 612.101081][ C1] R10: ffffed1015d06d11 R11: 0000000000000000 R12: dffffc0000000000 +[ 612.109241][ C1] R13: 0000000000000001 R14: 0000000000000000 R15: ffffc90001c27c08 +[ 612.145958][ C1] on_each_cpu+0x4d/0x1f0 +[ 612.150298][ C1] clock_was_set+0x18/0x20 +[ 612.154718][ C1] timekeeping_inject_offset+0x5e6/0x610 +[ 612.164988][ C1] do_adjtimex+0x3ac/0x8f0 +[ 612.174200][ C1] __x64_sys_clock_adjtime+0x17e/0x210 +[ 612.185765][ C1] do_syscall_64+0x73/0xe0 +[ 612.190187][ C1] entry_SYSCALL_64_after_hwframe+0x44/0xa9 +[ 612.196263][ C1] RIP: 0033:0x7f4117ebd1b7 +[ 612.200677][ C1] Code: Bad RIP value. +[ 612.204760][ C1] RSP: 002b:00007ffeedde6cd8 EFLAGS: 00000246 ORIG_RAX: 0000000000000131 +[ 612.213177][ C1] RAX: ffffffffffffffda RBX: 000055f707897630 RCX: 00007f4117ebd1b7 +[ 612.221151][ C1] RDX: ffffffffe7ce77f1 RSI: 00007ffeedde6da0 RDI: 0000000000000000 +[ 612.229141][ C1] RBP: 00007ffeedde6da0 R08: 0000000000000020 R09: 000000000000000d +[ 612.237138][ C1] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 612.245140][ C1] R13: 00007ffeedde6e90 R14: 0000000000000005 R15: 0000000000000001 +[ 613.358055][ C1] Shutting down cpus with NMI +[ 613.363983][ C1] Kernel Offset: disabled +[ 613.368409][ C1] Rebooting in 86400 seconds.. diff --git a/pkg/report/testdata/linux/report/503 b/pkg/report/testdata/linux/report/503 new file mode 100644 index 000000000..19d7078b5 --- /dev/null +++ b/pkg/report/testdata/linux/report/503 @@ -0,0 +1,141 @@ +TITLE: BUG: soft lockup in on_each_cpu +TYPE: HANG + +[ 1117.938056][ C0] watchdog: BUG: soft lockup - CPU#0 stuck for 123s! [syz-executor.1:2970] +[ 1117.946816][ C0] Modules linked in: +[ 1117.950702][ C0] irq event stamp: 29938 +[ 1117.954945][ C0] hardirqs last enabled at (29937): [] asm_sysvec_apic_timer_interrupt+0x12/0x20 +[ 1117.965695][ C0] hardirqs last disabled at (29938): [] idtentry_enter_cond_rcu+0x1d/0x50 +[ 1117.975747][ C0] softirqs last enabled at (24182): [] __do_softirq+0x748/0xa60 +[ 1117.985017][ C0] softirqs last disabled at (24173): [] asm_call_on_stack+0xf/0x20 +[ 1117.994982][ C0] CPU: 0 PID: 2970 Comm: syz-executor.1 Not tainted 5.8.0-rc3-syzkaller #0 +[ 1118.003612][ C0] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1118.013725][ C0] RIP: 0010:smp_call_function_single+0x192/0x4f0 +[ 1118.020047][ C0] Code: 10 8b 7c 24 1c 48 8d 74 24 40 48 89 44 24 50 48 8b 44 24 08 48 89 44 24 58 e8 fa f9 ff ff 41 89 c5 eb 07 e8 f0 ff 0a 00 f3 90 <44> 8b 64 24 48 31 ff 41 83 e4 01 44 89 e6 e8 5b fc 0a 00 45 85 e4 +[ 1118.039657][ C0] RSP: 0018:ffffc90004a27c00 EFLAGS: 00000293 +[ 1118.045722][ C0] RAX: 0000000000000000 RBX: 1ffff92000944f84 RCX: ffffffff8168b785 +[ 1118.053685][ C0] RDX: ffff88804f868500 RSI: ffffffff8168b770 RDI: 0000000000000005 +[ 1118.061649][ C0] RBP: ffffc90004a27cc8 R08: 0000000000000001 R09: ffff8880ae736dc7 +[ 1118.069615][ C0] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000001 +[ 1118.077576][ C0] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000040 +[ 1118.085543][ C0] FS: 00007fa61e16a700(0000) GS:ffff8880ae600000(0000) knlGS:0000000000000000 +[ 1118.094460][ C0] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 1118.101033][ C0] CR2: 00007fa61e127db8 CR3: 000000003e870000 CR4: 00000000001406f0 +[ 1118.108995][ C0] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 1118.116956][ C0] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 1118.124912][ C0] Call Trace: +[ 1118.160223][ C0] smp_call_function_many_cond+0x1a4/0x990 +[ 1118.183251][ C0] on_each_cpu+0x4a/0x240 +[ 1118.187582][ C0] clock_was_set+0x18/0x20 +[ 1118.191991][ C0] do_settimeofday64+0x350/0x4e0 +[ 1118.206352][ C0] do_sys_settimeofday64+0x1de/0x260 +[ 1118.211658][ C0] __x64_sys_clock_settime+0x197/0x260 +[ 1118.237309][ C0] do_syscall_64+0x60/0xe0 +[ 1118.241732][ C0] entry_SYSCALL_64_after_hwframe+0x44/0xa9 +[ 1118.247682][ C0] RIP: 0033:0x45cb29 +[ 1118.251571][ C0] Code: Bad RIP value. +[ 1118.255632][ C0] RSP: 002b:00007fa61e169c78 EFLAGS: 00000246 ORIG_RAX: 00000000000000e3 +[ 1118.264033][ C0] RAX: ffffffffffffffda RBX: 00000000004db560 RCX: 000000000045cb29 +[ 1118.271999][ C0] RDX: 0000000000000000 RSI: 0000000020000400 RDI: 0000000000000000 +[ 1118.279961][ C0] RBP: 000000000078bf00 R08: 0000000000000000 R09: 0000000000000000 +[ 1118.287921][ C0] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 1118.295880][ C0] R13: 000000000000007b R14: 00000000004c34ac R15: 00007fa61e16a6d4 +[ 1118.303861][ C0] Sending NMI from CPU 0 to CPUs 1: +[ 1118.311128][ C0] NMI backtrace for cpu 1 +[ 1118.311136][ C0] CPU: 1 PID: 2956 Comm: syz-executor.3 Not tainted 5.8.0-rc3-syzkaller #0 +[ 1118.311144][ C0] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1118.311149][ C0] RIP: 0010:__sanitizer_cov_trace_pc+0x0/0x60 +[ 1118.311164][ C0] Code: 48 89 ef 5d e9 01 07 3f 00 5d be 03 00 00 00 e9 26 b3 23 02 66 0f 1f 44 00 00 48 8b be b0 01 00 00 e8 b4 ff ff ff 31 c0 c3 90 <65> 48 8b 14 25 00 ff 01 00 65 8b 05 50 47 8e 7e a9 00 01 ff 00 48 +[ 1118.311169][ C0] RSP: 0018:ffffc90000da8de0 EFLAGS: 00000046 +[ 1118.311179][ C0] RAX: ffff888089f82480 RBX: ffff888089f82480 RCX: 0000000000000000 +[ 1118.311186][ C0] RDX: dffffc0000000000 RSI: ffffffff89bc1180 RDI: ffff888089f82d38 +[ 1118.311193][ C0] RBP: 0000000000000001 R08: 0000000000000001 R09: ffff888089f82d48 +[ 1118.311199][ C0] R10: fffffbfff155cb29 R11: 0000000000000000 R12: ffff8880ae727740 +[ 1118.311206][ C0] R13: ffff8880ae727840 R14: ffff888091f6e340 R15: dffffc0000000000 +[ 1118.311213][ C0] FS: 00007f7625d5c700(0000) GS:ffff8880ae700000(0000) knlGS:0000000000000000 +[ 1118.311219][ C0] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 1118.311226][ C0] CR2: 00007f68ff5c0240 CR3: 000000004c5a3000 CR4: 00000000001406e0 +[ 1118.311232][ C0] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 1118.311239][ C0] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 1118.311242][ C0] Call Trace: +[ 1118.311246][ C0] +[ 1118.311250][ C0] __hrtimer_run_queues+0x1de/0xfc0 +[ 1118.311266][ C0] hrtimer_interrupt+0x32a/0x930 +[ 1118.311275][ C0] __sysvec_apic_timer_interrupt+0x142/0x5e0 +[ 1118.311280][ C0] asm_call_on_stack+0xf/0x20 +[ 1118.311283][ C0] +[ 1118.311288][ C0] sysvec_apic_timer_interrupt+0xe0/0x120 +[ 1118.311293][ C0] asm_sysvec_apic_timer_interrupt+0x12/0x20 +[ 1118.311298][ C0] RIP: 0010:lock_acquire+0x270/0xad0 +[ 1118.311313][ C0] Code: 00 00 00 00 fc ff df 48 c1 e8 03 80 3c 10 00 0f 85 f8 06 00 00 48 83 3d 6d d0 5a 08 00 0f 84 a6 05 00 00 48 8b 7c 24 08 57 9d <0f> 1f 44 00 00 48 b8 00 00 00 00 00 fc ff df 48 03 44 24 10 48 c7 +[ 1118.311318][ C0] RSP: 0018:ffffc900049b7610 EFLAGS: 00000282 +[ 1118.311328][ C0] RAX: 1ffffffff1369c08 RBX: ffff888089f82480 RCX: 0000000000000001 +[ 1118.311334][ C0] RDX: dffffc0000000000 RSI: e297a4d1167c23bf RDI: 0000000000000282 +[ 1118.311341][ C0] RBP: 0000000000000000 R08: 0000000000000000 R09: ffffffff8c58fa27 +[ 1118.311348][ C0] R10: fffffbfff18b1f44 R11: 0000000000000000 R12: 0000000000000002 +[ 1118.311354][ C0] R13: ffffffff89bc1180 R14: 0000000000000000 R15: ffff8880975374a8 +[ 1118.311381][ C0] lock_page_memcg+0x63/0x260 +[ 1118.311390][ C0] page_remove_rmap+0x25/0x1670 +[ 1118.311395][ C0] unmap_page_range+0xf3b/0x2940 +[ 1118.311419][ C0] unmap_single_vma+0x198/0x300 +[ 1118.311423][ C0] unmap_vmas+0x16f/0x2f0 +[ 1118.311432][ C0] exit_mmap+0x2b1/0x510 +[ 1118.311455][ C0] __mmput+0x122/0x470 +[ 1118.311465][ C0] mmput+0x53/0x60 +[ 1118.311469][ C0] do_exit+0xa8f/0x2a40 +[ 1118.311501][ C0] do_group_exit+0x125/0x310 +[ 1118.311505][ C0] get_signal+0x40b/0x1ee0 +[ 1118.311519][ C0] do_signal+0x82/0x2520 +[ 1118.311569][ C0] __prepare_exit_to_usermode+0x156/0x1f0 +[ 1118.311573][ C0] do_syscall_64+0x6c/0xe0 +[ 1118.311578][ C0] entry_SYSCALL_64_after_hwframe+0x44/0xa9 +[ 1118.311582][ C0] RIP: 0033:0x45cb29 +[ 1118.311586][ C0] Code: Bad RIP value. +[ 1118.311591][ C0] RSP: 002b:00007f7625d5bc78 EFLAGS: 00000246 ORIG_RAX: 00000000000000a6 +[ 1118.311603][ C0] RAX: 0000000000000000 RBX: 000000000050b9e0 RCX: 000000000045cb29 +[ 1118.311609][ C0] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000020000500 +[ 1118.311616][ C0] RBP: 000000000078bfa0 R08: 0000000000000000 R09: 0000000000000000 +[ 1118.311623][ C0] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 1118.311630][ C0] R13: 0000000000000c8d R14: 00000000004ceea7 R15: 00007f7625d5c6d4 +[ 1118.311638][ C0] Kernel panic - not syncing: softlockup: hung tasks +[ 1118.872803][ C0] CPU: 0 PID: 2970 Comm: syz-executor.1 Tainted: G L 5.8.0-rc3-syzkaller #0 +[ 1118.882764][ C0] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1118.892807][ C0] Call Trace: +[ 1118.896082][ C0] +[ 1118.898932][ C0] dump_stack+0x18f/0x20d +[ 1118.903261][ C0] panic+0x2e3/0x75c +[ 1118.931713][ C0] watchdog_timer_fn.cold+0x16/0x2a +[ 1118.936909][ C0] __hrtimer_run_queues+0x1d5/0xfc0 +[ 1118.960114][ C0] hrtimer_interrupt+0x32a/0x930 +[ 1118.965067][ C0] __sysvec_apic_timer_interrupt+0x142/0x5e0 +[ 1118.971042][ C0] asm_call_on_stack+0xf/0x20 +[ 1118.975705][ C0] +[ 1118.978636][ C0] sysvec_apic_timer_interrupt+0xe0/0x120 +[ 1118.984347][ C0] asm_sysvec_apic_timer_interrupt+0x12/0x20 +[ 1118.990321][ C0] RIP: 0010:smp_call_function_single+0x192/0x4f0 +[ 1118.996764][ C0] Code: 10 8b 7c 24 1c 48 8d 74 24 40 48 89 44 24 50 48 8b 44 24 08 48 89 44 24 58 e8 fa f9 ff ff 41 89 c5 eb 07 e8 f0 ff 0a 00 f3 90 <44> 8b 64 24 48 31 ff 41 83 e4 01 44 89 e6 e8 5b fc 0a 00 45 85 e4 +[ 1119.016378][ C0] RSP: 0018:ffffc90004a27c00 EFLAGS: 00000293 +[ 1119.022572][ C0] RAX: 0000000000000000 RBX: 1ffff92000944f84 RCX: ffffffff8168b785 +[ 1119.030547][ C0] RDX: ffff88804f868500 RSI: ffffffff8168b770 RDI: 0000000000000005 +[ 1119.038530][ C0] RBP: ffffc90004a27cc8 R08: 0000000000000001 R09: ffff8880ae736dc7 +[ 1119.046501][ C0] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000001 +[ 1119.054478][ C0] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000040 +[ 1119.105952][ C0] smp_call_function_many_cond+0x1a4/0x990 +[ 1119.128997][ C0] on_each_cpu+0x4a/0x240 +[ 1119.133319][ C0] clock_was_set+0x18/0x20 +[ 1119.137730][ C0] do_settimeofday64+0x350/0x4e0 +[ 1119.152025][ C0] do_sys_settimeofday64+0x1de/0x260 +[ 1119.157303][ C0] __x64_sys_clock_settime+0x197/0x260 +[ 1119.182915][ C0] do_syscall_64+0x60/0xe0 +[ 1119.187325][ C0] entry_SYSCALL_64_after_hwframe+0x44/0xa9 +[ 1119.193228][ C0] RIP: 0033:0x45cb29 +[ 1119.197104][ C0] Code: Bad RIP value. +[ 1119.201158][ C0] RSP: 002b:00007fa61e169c78 EFLAGS: 00000246 ORIG_RAX: 00000000000000e3 +[ 1119.209558][ C0] RAX: ffffffffffffffda RBX: 00000000004db560 RCX: 000000000045cb29 +[ 1119.217519][ C0] RDX: 0000000000000000 RSI: 0000000020000400 RDI: 0000000000000000 +[ 1119.225478][ C0] RBP: 000000000078bf00 R08: 0000000000000000 R09: 0000000000000000 +[ 1119.233440][ C0] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 1119.241398][ C0] R13: 000000000000007b R14: 00000000004c34ac R15: 00007fa61e16a6d4 +[ 1120.387767][ C0] Shutting down cpus with NMI +[ 1120.393794][ C0] Kernel Offset: disabled +[ 1120.398124][ C0] Rebooting in 86400 seconds.. diff --git a/pkg/report/testdata/linux/report/504 b/pkg/report/testdata/linux/report/504 new file mode 100644 index 000000000..5b15d3fe7 --- /dev/null +++ b/pkg/report/testdata/linux/report/504 @@ -0,0 +1,163 @@ +TITLE: BUG: soft lockup in on_each_cpu +TYPE: HANG + +[ 635.060078][ C0] watchdog: BUG: soft lockup - CPU#0 stuck for 123s! [syz-executor.2:12834] +[ 635.068861][ C0] Modules linked in: +[ 635.073526][ C0] irq event stamp: 30574 +[ 635.077838][ C0] hardirqs last enabled at (30573): [] asm_sysvec_apic_timer_interrupt+0x12/0x20 +[ 635.088590][ C0] hardirqs last disabled at (30574): [] idtentry_enter_cond_rcu+0x1d/0x50 +[ 635.098646][ C0] softirqs last enabled at (13814): [] __do_softirq+0x748/0xa60 +[ 635.107920][ C0] softirqs last disabled at (13753): [] asm_call_on_stack+0xf/0x20 +[ 635.118057][ C0] CPU: 0 PID: 12834 Comm: syz-executor.2 Not tainted 5.8.0-rc3-syzkaller #0 +[ 635.126714][ C0] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 635.136820][ C0] RIP: 0010:smp_call_function_single+0x197/0x4f0 +[ 635.143138][ C0] Code: 48 8d 74 24 40 48 89 44 24 50 48 8b 44 24 08 48 89 44 24 58 e8 fa f9 ff ff 41 89 c5 eb 07 e8 f0 ff 0a 00 f3 90 44 8b 64 24 48 <31> ff 41 83 e4 01 44 89 e6 e8 5b fc 0a 00 45 85 e4 75 e1 e8 d1 ff +[ 635.162818][ C0] RSP: 0018:ffffc9000466f5e0 EFLAGS: 00000293 +[ 635.168970][ C0] RAX: 0000000000000000 RBX: 1ffff920008cdec0 RCX: ffffffff8168b785 +[ 635.179534][ C0] RDX: ffff88804550c200 RSI: ffffffff8168b770 RDI: 0000000000000005 +[ 635.187589][ C0] RBP: ffffc9000466f6b8 R08: 0000000000000001 R09: ffff8880ae736dc7 +[ 635.195551][ C0] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000011 +[ 635.203540][ C0] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000040 +[ 635.211850][ C0] FS: 00007fc9cbaaa700(0000) GS:ffff8880ae600000(0000) knlGS:0000000000000000 +[ 635.220983][ C0] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 635.227827][ C0] CR2: 000000c00005bc80 CR3: 00000000960ba000 CR4: 00000000001406f0 +[ 635.235812][ C0] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 635.244211][ C0] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 635.252431][ C0] Call Trace: +[ 635.286610][ C0] smp_call_function_many_cond+0x1a4/0x990 +[ 635.312583][ C0] on_each_cpu+0x4a/0x240 +[ 635.316906][ C0] text_poke_bp_batch+0x142/0x3d0 +[ 635.338876][ C0] text_poke_finish+0x16/0x21 +[ 635.343543][ C0] arch_jump_label_transform_apply+0x13/0x20 +[ 635.349511][ C0] jump_label_update+0x1b3/0x3a0 +[ 635.354469][ C0] __static_key_slow_dec_cpuslocked.part.0+0x59/0x60 +[ 635.361135][ C0] static_key_slow_dec+0x78/0xc0 +[ 635.366106][ C0] tracepoint_probe_unregister+0x4fb/0x6b0 +[ 635.382469][ C0] trace_event_reg+0x181/0x350 +[ 635.393733][ C0] perf_trace_event_unreg.isra.0+0xaa/0x200 +[ 635.399656][ C0] perf_trace_destroy+0xb5/0xf0 +[ 635.409699][ C0] _free_event+0x2ee/0x11e0 +[ 635.414292][ C0] perf_event_release_kernel+0xa24/0xe00 +[ 635.436630][ C0] perf_release+0x33/0x40 +[ 635.441181][ C0] __fput+0x33c/0x880 +[ 635.445477][ C0] task_work_run+0xdd/0x190 +[ 635.450012][ C0] do_exit+0xb72/0x2a40 +[ 635.485151][ C0] do_group_exit+0x125/0x310 +[ 635.490691][ C0] get_signal+0x40b/0x1ee0 +[ 635.502065][ C0] do_signal+0x82/0x2520 +[ 635.555253][ C0] __prepare_exit_to_usermode+0x156/0x1f0 +[ 635.561518][ C0] do_syscall_64+0x6c/0xe0 +[ 635.566462][ C0] entry_SYSCALL_64_after_hwframe+0x44/0xa9 +[ 635.572837][ C0] RIP: 0033:0x45cb29 +[ 635.576820][ C0] Code: Bad RIP value. +[ 635.581756][ C0] RSP: 002b:00007fc9cbaa9cf8 EFLAGS: 00000246 ORIG_RAX: 00000000000000ca +[ 635.590331][ C0] RAX: fffffffffffffe00 RBX: 000000000078bfa8 RCX: 000000000045cb29 +[ 635.598291][ C0] RDX: 0000000000000000 RSI: 0000000000000080 RDI: 000000000078bfa8 +[ 635.606254][ C0] RBP: 000000000078bfa0 R08: 0000000000000000 R09: 0000000000000000 +[ 635.614217][ C0] R10: 0000000000000000 R11: 0000000000000246 R12: 000000000078bfac +[ 635.622176][ C0] R13: 00007fff67a9f90f R14: 00007fc9cbaaa9c0 R15: 000000000078bfac +[ 635.630260][ C0] Sending NMI from CPU 0 to CPUs 1: +[ 635.636609][ C0] NMI backtrace for cpu 1 +[ 635.636616][ C0] CPU: 1 PID: 13865 Comm: syz-executor.0 Not tainted 5.8.0-rc3-syzkaller #0 +[ 635.636625][ C0] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 635.636630][ C0] RIP: 0010:_raw_spin_unlock_irqrestore+0x51/0xe0 +[ 635.636645][ C0] Code: e0 b4 89 48 ba 00 00 00 00 00 fc ff df 48 c1 e8 03 80 3c 10 00 0f 85 80 00 00 00 48 83 3d 36 a7 cd 01 00 74 5f 48 89 df 57 9d <0f> 1f 44 00 00 e8 05 2d 92 f9 eb 34 e8 5e 33 92 f9 48 c7 c0 40 e0 +[ 635.636650][ C0] RSP: 0018:ffffc90000da8dd0 EFLAGS: 00000082 +[ 635.636660][ C0] RAX: 1ffffffff1369c08 RBX: 0000000000000082 RCX: ffffffff815aff60 +[ 635.636667][ C0] RDX: dffffc0000000000 RSI: 0000000000000004 RDI: 0000000000000082 +[ 635.636673][ C0] RBP: ffff8880ae727740 R08: 0000000000000000 R09: ffff8880ae727743 +[ 635.636680][ C0] R10: ffffed1015ce4ee8 R11: 0000000000000000 R12: ffff8880ae727740 +[ 635.636686][ C0] R13: ffff8880ae727840 R14: ffff88808fac9b40 R15: dffffc0000000000 +[ 635.636694][ C0] FS: 00007f86e73d9700(0000) GS:ffff8880ae700000(0000) knlGS:0000000000000000 +[ 635.636699][ C0] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 635.636706][ C0] CR2: 0000001b2e421000 CR3: 000000004b7e9000 CR4: 00000000001406e0 +[ 635.636713][ C0] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 635.636719][ C0] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 635.636723][ C0] Call Trace: +[ 635.636725][ C0] +[ 635.636730][ C0] __hrtimer_run_queues+0x5d1/0xfc0 +[ 635.636750][ C0] hrtimer_interrupt+0x32a/0x930 +[ 635.636755][ C0] __sysvec_apic_timer_interrupt+0x142/0x5e0 +[ 635.636759][ C0] asm_call_on_stack+0xf/0x20 +[ 635.636762][ C0] +[ 635.636767][ C0] sysvec_apic_timer_interrupt+0xe0/0x120 +[ 635.636773][ C0] asm_sysvec_apic_timer_interrupt+0x12/0x20 +[ 635.636778][ C0] RIP: 0010:_raw_spin_unlock_irqrestore+0x8c/0xe0 +[ 635.636793][ C0] Code: 48 c7 c0 40 e0 b4 89 48 ba 00 00 00 00 00 fc ff df 48 c1 e8 03 80 3c 10 00 75 37 48 83 3d fb a6 cd 01 00 74 22 48 89 df 57 9d <0f> 1f 44 00 00 bf 01 00 00 00 e8 35 39 68 f9 65 8b 05 5e c5 1a 78 +[ 635.636798][ C0] RSP: 0018:ffffc90006edfd80 EFLAGS: 00000282 +[ 635.636807][ C0] RAX: 1ffffffff1369c08 RBX: 0000000000000282 RCX: 0000000000000000 +[ 635.636814][ C0] RDX: dffffc0000000000 RSI: 0000000000000000 RDI: 0000000000000282 +[ 635.636820][ C0] RBP: ffffffff89bd19e0 R08: 0000000000000000 R09: 0000000000000000 +[ 635.636827][ C0] R10: 0000000000000001 R11: 0000000000000000 R12: 1ffff92000ddbfbc +[ 635.636834][ C0] R13: 0000000000000000 R14: ffffc90006edfed0 R15: ffffffffa0fc92c3 +[ 635.636838][ C0] do_settimeofday64+0x34b/0x4e0 +[ 635.636851][ C0] do_sys_settimeofday64+0x1de/0x260 +[ 635.636856][ C0] __x64_sys_clock_settime+0x197/0x260 +[ 635.636878][ C0] do_syscall_64+0x60/0xe0 +[ 635.636884][ C0] entry_SYSCALL_64_after_hwframe+0x44/0xa9 +[ 635.636887][ C0] RIP: 0033:0x45cb29 +[ 635.636891][ C0] Code: Bad RIP value. +[ 635.636896][ C0] RSP: 002b:00007f86e73d8c78 EFLAGS: 00000246 ORIG_RAX: 00000000000000e3 +[ 635.636907][ C0] RAX: ffffffffffffffda RBX: 00000000004db560 RCX: 000000000045cb29 +[ 635.636914][ C0] RDX: 0000000000000000 RSI: 0000000020000400 RDI: 0000000000000000 +[ 635.636921][ C0] RBP: 000000000078bf00 R08: 0000000000000000 R09: 0000000000000000 +[ 635.636927][ C0] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 635.636934][ C0] R13: 000000000000007b R14: 00000000004c34ac R15: 00007f86e73d96d4 +[ 635.636942][ C0] Kernel panic - not syncing: softlockup: hung tasks +[ 636.023252][ C0] CPU: 0 PID: 12834 Comm: syz-executor.2 Tainted: G L 5.8.0-rc3-syzkaller #0 +[ 636.033299][ C0] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 636.043351][ C0] Call Trace: +[ 636.046636][ C0] +[ 636.049608][ C0] dump_stack+0x18f/0x20d +[ 636.055303][ C0] panic+0x2e3/0x75c +[ 636.083775][ C0] watchdog_timer_fn.cold+0x16/0x2a +[ 636.088971][ C0] __hrtimer_run_queues+0x1d5/0xfc0 +[ 636.112471][ C0] hrtimer_interrupt+0x32a/0x930 +[ 636.117503][ C0] __sysvec_apic_timer_interrupt+0x142/0x5e0 +[ 636.123476][ C0] asm_call_on_stack+0xf/0x20 +[ 636.128194][ C0] +[ 636.131126][ C0] sysvec_apic_timer_interrupt+0xe0/0x120 +[ 636.136836][ C0] asm_sysvec_apic_timer_interrupt+0x12/0x20 +[ 636.142808][ C0] RIP: 0010:smp_call_function_single+0x197/0x4f0 +[ 636.149123][ C0] Code: 48 8d 74 24 40 48 89 44 24 50 48 8b 44 24 08 48 89 44 24 58 e8 fa f9 ff ff 41 89 c5 eb 07 e8 f0 ff 0a 00 f3 90 44 8b 64 24 48 <31> ff 41 83 e4 01 44 89 e6 e8 5b fc 0a 00 45 85 e4 75 e1 e8 d1 ff +[ 636.170465][ C0] RSP: 0018:ffffc9000466f5e0 EFLAGS: 00000293 +[ 636.176525][ C0] RAX: 0000000000000000 RBX: 1ffff920008cdec0 RCX: ffffffff8168b785 +[ 636.184487][ C0] RDX: ffff88804550c200 RSI: ffffffff8168b770 RDI: 0000000000000005 +[ 636.192447][ C0] RBP: ffffc9000466f6b8 R08: 0000000000000001 R09: ffff8880ae736dc7 +[ 636.200407][ C0] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000011 +[ 636.208368][ C0] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000040 +[ 636.259397][ C0] smp_call_function_many_cond+0x1a4/0x990 +[ 636.285749][ C0] on_each_cpu+0x4a/0x240 +[ 636.291120][ C0] text_poke_bp_batch+0x142/0x3d0 +[ 636.315309][ C0] text_poke_finish+0x16/0x21 +[ 636.319978][ C0] arch_jump_label_transform_apply+0x13/0x20 +[ 636.325956][ C0] jump_label_update+0x1b3/0x3a0 +[ 636.330978][ C0] __static_key_slow_dec_cpuslocked.part.0+0x59/0x60 +[ 636.337645][ C0] static_key_slow_dec+0x78/0xc0 +[ 636.342581][ C0] tracepoint_probe_unregister+0x4fb/0x6b0 +[ 636.360775][ C0] trace_event_reg+0x181/0x350 +[ 636.371429][ C0] perf_trace_event_unreg.isra.0+0xaa/0x200 +[ 636.377327][ C0] perf_trace_destroy+0xb5/0xf0 +[ 636.390247][ C0] _free_event+0x2ee/0x11e0 +[ 636.394754][ C0] perf_event_release_kernel+0xa24/0xe00 +[ 636.416956][ C0] perf_release+0x33/0x40 +[ 636.421297][ C0] __fput+0x33c/0x880 +[ 636.425544][ C0] task_work_run+0xdd/0x190 +[ 636.430048][ C0] do_exit+0xb72/0x2a40 +[ 636.463457][ C0] do_group_exit+0x125/0x310 +[ 636.468045][ C0] get_signal+0x40b/0x1ee0 +[ 636.477693][ C0] do_signal+0x82/0x2520 +[ 636.523170][ C0] __prepare_exit_to_usermode+0x156/0x1f0 +[ 636.528884][ C0] do_syscall_64+0x6c/0xe0 +[ 636.533312][ C0] entry_SYSCALL_64_after_hwframe+0x44/0xa9 +[ 636.539198][ C0] RIP: 0033:0x45cb29 +[ 636.543082][ C0] Code: Bad RIP value. +[ 636.547139][ C0] RSP: 002b:00007fc9cbaa9cf8 EFLAGS: 00000246 ORIG_RAX: 00000000000000ca +[ 636.556948][ C0] RAX: fffffffffffffe00 RBX: 000000000078bfa8 RCX: 000000000045cb29 +[ 636.565278][ C0] RDX: 0000000000000000 RSI: 0000000000000080 RDI: 000000000078bfa8 +[ 636.573338][ C0] RBP: 000000000078bfa0 R08: 0000000000000000 R09: 0000000000000000 +[ 636.581396][ C0] R10: 0000000000000000 R11: 0000000000000246 R12: 000000000078bfac +[ 636.589558][ C0] R13: 00007fff67a9f90f R14: 00007fc9cbaaa9c0 R15: 000000000078bfac +[ 637.736857][ C0] Shutting down cpus with NMI +[ 637.743185][ C0] Kernel Offset: disabled +[ 637.747556][ C0] Rebooting in 86400 seconds.. diff --git a/pkg/report/testdata/linux/report/505 b/pkg/report/testdata/linux/report/505 new file mode 100644 index 000000000..1ea43125c --- /dev/null +++ b/pkg/report/testdata/linux/report/505 @@ -0,0 +1,151 @@ +TITLE: BUG: soft lockup in on_each_cpu +TYPE: HANG + +[ 1472.340932][ C1] watchdog: BUG: soft lockup - CPU#1 stuck for 123s! [syz-executor.0:8399] +[ 1472.349674][ C1] Modules linked in: +[ 1472.353561][ C1] irq event stamp: 14624782 +[ 1472.358138][ C1] hardirqs last enabled at (14624781): [] asm_sysvec_apic_timer_interrupt+0x12/0x20 +[ 1472.369259][ C1] hardirqs last disabled at (14624782): [] idtentry_enter_cond_rcu+0x1d/0x50 +[ 1472.379577][ C1] softirqs last enabled at (1000602): [] __do_softirq+0x748/0xa60 +[ 1472.389029][ C1] softirqs last disabled at (1000171): [] asm_call_on_stack+0xf/0x20 +[ 1472.399261][ C1] CPU: 1 PID: 8399 Comm: syz-executor.0 Not tainted 5.8.0-rc3-syzkaller #0 +[ 1472.407829][ C1] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1472.417909][ C1] RIP: 0010:__sanitizer_cov_trace_pc+0x19/0x60 +[ 1472.424147][ C1] Code: 00 48 8b be b0 01 00 00 e8 b4 ff ff ff 31 c0 c3 90 65 48 8b 14 25 00 ff 01 00 65 8b 05 90 47 8e 7e a9 00 01 ff 00 48 8b 34 24 <74> 0f f6 c4 01 74 35 8b 82 04 14 00 00 85 c0 74 2b 8b 82 e0 13 00 +[ 1472.443866][ C1] RSP: 0018:ffffc90006e0f598 EFLAGS: 00000246 +[ 1472.449938][ C1] RAX: 0000000000000003 RBX: 1ffff92000dc1eb8 RCX: ffffffff8168b745 +[ 1472.457901][ C1] RDX: ffff88803b2ba200 RSI: ffffffff8168b730 RDI: 0000000000000005 +[ 1472.465861][ C1] RBP: ffffc90006e0f670 R08: 0000000000000001 R09: ffff8880ae636dc7 +[ 1472.473826][ C1] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000001 +[ 1472.481790][ C1] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000040 +[ 1472.489757][ C1] FS: 00007fbf41e8e700(0000) GS:ffff8880ae700000(0000) knlGS:0000000000000000 +[ 1472.498676][ C1] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 1472.505251][ C1] CR2: 00007f4ca5364f87 CR3: 00000000a83c6000 CR4: 00000000001406e0 +[ 1472.513215][ C1] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 1472.521183][ C1] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000600 +[ 1472.529139][ C1] Call Trace: +[ 1472.532426][ C1] smp_call_function_single+0x190/0x4f0 +[ 1472.570082][ C1] smp_call_function_many_cond+0x1a4/0x990 +[ 1472.591361][ C1] on_each_cpu+0x4a/0x240 +[ 1472.595685][ C1] __purge_vmap_area_lazy+0x11e/0x1f20 +[ 1472.611965][ C1] _vm_unmap_aliases.part.0+0x3cc/0x4d0 +[ 1472.617525][ C1] vm_unmap_aliases+0x42/0x50 +[ 1472.622198][ C1] change_page_attr_set_clr+0x241/0x500 +[ 1472.643456][ C1] set_memory_ro+0x78/0xa0 +[ 1472.662220][ C1] bpf_int_jit_compile+0xd84/0x110c +[ 1472.678173][ C1] bpf_prog_select_runtime+0x591/0xb00 +[ 1472.689254][ C1] bpf_migrate_filter+0x2dc/0x380 +[ 1472.704588][ C1] __get_filter+0x357/0x4e0 +[ 1472.709105][ C1] sk_attach_filter+0x1c/0x170 +[ 1472.713865][ C1] sock_setsockopt+0x1f7b/0x22a0 +[ 1472.734671][ C1] __sys_setsockopt+0x41a/0x480 +[ 1472.756467][ C1] __x64_sys_setsockopt+0xba/0x150 +[ 1472.766681][ C1] do_syscall_64+0x60/0xe0 +[ 1472.771093][ C1] entry_SYSCALL_64_after_hwframe+0x44/0xa9 +[ 1472.776978][ C1] RIP: 0033:0x45cb29 +[ 1472.780871][ C1] Code: Bad RIP value. +[ 1472.784939][ C1] RSP: 002b:00007fbf41e8dc78 EFLAGS: 00000246 ORIG_RAX: 0000000000000036 +[ 1472.793340][ C1] RAX: ffffffffffffffda RBX: 0000000000503c60 RCX: 000000000045cb29 +[ 1472.801301][ C1] RDX: 000000000000001a RSI: 0000000000000001 RDI: 0000000000000004 +[ 1472.809264][ C1] RBP: 000000000078bf00 R08: 0000000000000010 R09: 0000000000000000 +[ 1472.817224][ C1] R10: 0000000020000140 R11: 0000000000000246 R12: 00000000ffffffff +[ 1472.825190][ C1] R13: 0000000000000a98 R14: 00000000004cd71b R15: 00007fbf41e8e6d4 +[ 1472.833175][ C1] Sending NMI from CPU 1 to CPUs 0: +[ 1472.839445][ C1] NMI backtrace for cpu 0 +[ 1472.839452][ C1] CPU: 0 PID: 8401 Comm: syz-executor.2 Not tainted 5.8.0-rc3-syzkaller #0 +[ 1472.839459][ C1] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1472.839464][ C1] RIP: 0010:__sanitizer_cov_trace_pc+0x0/0x60 +[ 1472.839476][ C1] Code: 48 89 ef 5d e9 01 07 3f 00 5d be 03 00 00 00 e9 a6 b2 23 02 66 0f 1f 44 00 00 48 8b be b0 01 00 00 e8 b4 ff ff ff 31 c0 c3 90 <65> 48 8b 14 25 00 ff 01 00 65 8b 05 90 47 8e 7e a9 00 01 ff 00 48 +[ 1472.839481][ C1] RSP: 0018:ffffc90000007de0 EFLAGS: 00000002 +[ 1472.839490][ C1] RAX: 0000000080010002 RBX: 0000000000000001 RCX: ffffffff81640ea3 +[ 1472.839496][ C1] RDX: 0000000000000001 RSI: ffff88805fe025c0 RDI: 0000000000000005 +[ 1472.839502][ C1] RBP: 0000000000000001 R08: 0000000000000000 R09: ffffffff8aae584f +[ 1472.839508][ C1] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8880ae627740 +[ 1472.839513][ C1] R13: ffff8880ae627940 R14: ffff888094ccab40 R15: dffffc0000000000 +[ 1472.839520][ C1] FS: 00007f91e4387700(0000) GS:ffff8880ae600000(0000) knlGS:0000000000000000 +[ 1472.839525][ C1] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 1472.839531][ C1] CR2: 0000001b30e22000 CR3: 000000005c19d000 CR4: 00000000001406f0 +[ 1472.839537][ C1] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 1472.839542][ C1] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000600 +[ 1472.839546][ C1] Call Trace: +[ 1472.839549][ C1] +[ 1472.839553][ C1] __hrtimer_run_queues+0xb3c/0xfc0 +[ 1472.839567][ C1] hrtimer_interrupt+0x32a/0x930 +[ 1472.839572][ C1] __sysvec_apic_timer_interrupt+0x142/0x5e0 +[ 1472.839576][ C1] asm_call_on_stack+0xf/0x20 +[ 1472.839579][ C1] +[ 1472.839584][ C1] sysvec_apic_timer_interrupt+0xe0/0x120 +[ 1472.839589][ C1] asm_sysvec_apic_timer_interrupt+0x12/0x20 +[ 1472.839594][ C1] RIP: 0010:_raw_spin_unlock_irqrestore+0x8c/0xe0 +[ 1472.839606][ C1] Code: 48 c7 c0 40 e0 b4 89 48 ba 00 00 00 00 00 fc ff df 48 c1 e8 03 80 3c 10 00 75 37 48 83 3d fb a6 cd 01 00 74 22 48 89 df 57 9d <0f> 1f 44 00 00 bf 01 00 00 00 e8 f5 38 68 f9 65 8b 05 5e c5 1a 78 +[ 1472.839610][ C1] RSP: 0018:ffffc90005bd7d80 EFLAGS: 00000282 +[ 1472.839619][ C1] RAX: 1ffffffff1369c08 RBX: 0000000000000282 RCX: 0000000000000000 +[ 1472.839625][ C1] RDX: dffffc0000000000 RSI: 0000000000000000 RDI: 0000000000000282 +[ 1472.839631][ C1] RBP: ffffffff89bd19a0 R08: 0000000000000000 R09: 0000000000000000 +[ 1472.839637][ C1] R10: 0000000000000001 R11: 0000000000000000 R12: 1ffff92000b7afbc +[ 1472.839643][ C1] R13: 0000000000000000 R14: ffffc90005bd7ed0 R15: ffffffffa102b94c +[ 1472.839647][ C1] do_settimeofday64+0x34b/0x4e0 +[ 1472.839663][ C1] do_sys_settimeofday64+0x1de/0x260 +[ 1472.839668][ C1] __x64_sys_clock_settime+0x197/0x260 +[ 1472.839689][ C1] do_syscall_64+0x60/0xe0 +[ 1472.839694][ C1] entry_SYSCALL_64_after_hwframe+0x44/0xa9 +[ 1472.839697][ C1] RIP: 0033:0x45cb29 +[ 1472.839701][ C1] Code: Bad RIP value. +[ 1472.839706][ C1] RSP: 002b:00007f91e4386c78 EFLAGS: 00000246 ORIG_RAX: 00000000000000e3 +[ 1472.839716][ C1] RAX: ffffffffffffffda RBX: 00000000004db520 RCX: 000000000045cb29 +[ 1472.839722][ C1] RDX: 0000000000000000 RSI: 0000000020000400 RDI: 0000000000000000 +[ 1472.839728][ C1] RBP: 000000000078bf00 R08: 0000000000000000 R09: 0000000000000000 +[ 1472.839734][ C1] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 1472.839739][ C1] R13: 000000000000007b R14: 00000000004c34ac R15: 00007f91e43876d4 +[ 1472.839750][ C1] Kernel panic - not syncing: softlockup: hung tasks +[ 1473.222485][ C1] CPU: 1 PID: 8399 Comm: syz-executor.0 Tainted: G L 5.8.0-rc3-syzkaller #0 +[ 1473.232445][ C1] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1473.242487][ C1] Call Trace: +[ 1473.245768][ C1] +[ 1473.248623][ C1] dump_stack+0x18f/0x20d +[ 1473.252970][ C1] panic+0x2e3/0x75c +[ 1473.281437][ C1] watchdog_timer_fn.cold+0x16/0x2a +[ 1473.286635][ C1] __hrtimer_run_queues+0x1d5/0xfc0 +[ 1473.310221][ C1] hrtimer_interrupt+0x32a/0x930 +[ 1473.315177][ C1] __sysvec_apic_timer_interrupt+0x142/0x5e0 +[ 1473.321156][ C1] asm_call_on_stack+0xf/0x20 +[ 1473.325820][ C1] +[ 1473.328761][ C1] sysvec_apic_timer_interrupt+0xe0/0x120 +[ 1473.334477][ C1] asm_sysvec_apic_timer_interrupt+0x12/0x20 +[ 1473.340471][ C1] RIP: 0010:__sanitizer_cov_trace_pc+0x19/0x60 +[ 1473.346623][ C1] Code: 00 48 8b be b0 01 00 00 e8 b4 ff ff ff 31 c0 c3 90 65 48 8b 14 25 00 ff 01 00 65 8b 05 90 47 8e 7e a9 00 01 ff 00 48 8b 34 24 <74> 0f f6 c4 01 74 35 8b 82 04 14 00 00 85 c0 74 2b 8b 82 e0 13 00 +[ 1473.366325][ C1] RSP: 0018:ffffc90006e0f598 EFLAGS: 00000246 +[ 1473.372385][ C1] RAX: 0000000000000003 RBX: 1ffff92000dc1eb8 RCX: ffffffff8168b745 +[ 1473.380347][ C1] RDX: ffff88803b2ba200 RSI: ffffffff8168b730 RDI: 0000000000000005 +[ 1473.388317][ C1] RBP: ffffc90006e0f670 R08: 0000000000000001 R09: ffff8880ae636dc7 +[ 1473.396282][ C1] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000001 +[ 1473.404268][ C1] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000040 +[ 1473.423701][ C1] smp_call_function_single+0x190/0x4f0 +[ 1473.461273][ C1] smp_call_function_many_cond+0x1a4/0x990 +[ 1473.482516][ C1] on_each_cpu+0x4a/0x240 +[ 1473.486842][ C1] __purge_vmap_area_lazy+0x11e/0x1f20 +[ 1473.503122][ C1] _vm_unmap_aliases.part.0+0x3cc/0x4d0 +[ 1473.508680][ C1] vm_unmap_aliases+0x42/0x50 +[ 1473.513351][ C1] change_page_attr_set_clr+0x241/0x500 +[ 1473.534585][ C1] set_memory_ro+0x78/0xa0 +[ 1473.553303][ C1] bpf_int_jit_compile+0xd84/0x110c +[ 1473.569185][ C1] bpf_prog_select_runtime+0x591/0xb00 +[ 1473.580220][ C1] bpf_migrate_filter+0x2dc/0x380 +[ 1473.595550][ C1] __get_filter+0x357/0x4e0 +[ 1473.600050][ C1] sk_attach_filter+0x1c/0x170 +[ 1473.604811][ C1] sock_setsockopt+0x1f7b/0x22a0 +[ 1473.625459][ C1] __sys_setsockopt+0x41a/0x480 +[ 1473.647223][ C1] __x64_sys_setsockopt+0xba/0x150 +[ 1473.657433][ C1] do_syscall_64+0x60/0xe0 +[ 1473.661846][ C1] entry_SYSCALL_64_after_hwframe+0x44/0xa9 +[ 1473.667801][ C1] RIP: 0033:0x45cb29 +[ 1473.671683][ C1] Code: Bad RIP value. +[ 1473.675742][ C1] RSP: 002b:00007fbf41e8dc78 EFLAGS: 00000246 ORIG_RAX: 0000000000000036 +[ 1473.684144][ C1] RAX: ffffffffffffffda RBX: 0000000000503c60 RCX: 000000000045cb29 +[ 1473.692112][ C1] RDX: 000000000000001a RSI: 0000000000000001 RDI: 0000000000000004 +[ 1473.700082][ C1] RBP: 000000000078bf00 R08: 0000000000000010 R09: 0000000000000000 +[ 1473.708046][ C1] R10: 0000000020000140 R11: 0000000000000246 R12: 00000000ffffffff +[ 1473.716022][ C1] R13: 0000000000000a98 R14: 00000000004cd71b R15: 00007fbf41e8e6d4 +[ 1474.819459][ C1] Shutting down cpus with NMI +[ 1474.825251][ C1] Kernel Offset: disabled +[ 1474.829603][ C1] Rebooting in 86400 seconds.. diff --git a/pkg/report/testdata/linux/report/65 b/pkg/report/testdata/linux/report/65 index 4e39b0278..2f1788305 100644 --- a/pkg/report/testdata/linux/report/65 +++ b/pkg/report/testdata/linux/report/65 @@ -1,4 +1,4 @@ -TITLE: BUG: soft lockup in jump_label_update_timeout +TITLE: BUG: soft lockup in on_each_cpu TYPE: HANG [ 247.938942] watchdog: BUG: soft lockup - CPU#0 stuck for 134s! [kworker/0:2:1400] -- cgit mrf-deployment