diff options
| author | Dmitry Vyukov <dvyukov@google.com> | 2018-09-09 19:19:00 +0200 |
|---|---|---|
| committer | Dmitry Vyukov <dvyukov@google.com> | 2018-09-10 16:19:47 +0200 |
| commit | b692332724338ba1034bcef46e3d05ba867d5d41 (patch) | |
| tree | f1e8d64f400eb0b1b0c58de72f49fe74bd01b5f2 /pkg | |
| parent | 58f18e3f60bc835d73ff566ccdfa3ad36b3b72a8 (diff) | |
pkg/report: improve rcu stall/lockup reports
During rcu stalls and cpu lockups kernel loops in some part of code,
usually across several functions. When the stall is detected, traceback
points to a random stack within the looping code. We generally take
the top function in the stack (with few exceptions) as the bug identity.
As the result stalls with the same root would produce multiple reports
in different functions, which is bad.
Instead we identify a representative function deeper in the stack.
For most syscalls it can be the syscall entry function (e.g. SyS_timer_create).
However, for highly discriminated functions syscalls like ioctl/read/write/connect
we take the previous function (e.g. for connect the one that points to exact
protocol, or for ioctl the one that is related to the device).
Fixes #710
Diffstat (limited to 'pkg')
48 files changed, 2218 insertions, 494 deletions
diff --git a/pkg/report/linux.go b/pkg/report/linux.go index 0513ffccc..182cd645e 100644 --- a/pkg/report/linux.go +++ b/pkg/report/linux.go @@ -504,6 +504,103 @@ func (ctx *linux) isCorrupted(title string, report []byte, format oopsFormat) (b return false, "" } +func linuxStallFrameExtractor(frames []string) (string, string) { + // During rcu stalls and cpu lockups kernel loops in some part of code, + // usually across several functions. When the stall is detected, traceback + // points to a random stack within the looping code. We generally take + // the top function in the stack (with few exceptions) as the bug identity. + // As the result stalls with the same root would produce multiple reports + // in different functions, which is bad. + // Instead we identify a representative function deeper in the stack. + // For most syscalls it can be the syscall entry function (e.g. SyS_timer_create). + // However, for highly discriminated functions syscalls like ioctl/read/write/connect + // we take the previous function (e.g. for connect the one that points to exact + // protocol, or for ioctl the one that is related to the device). + prev := frames[0] + for _, frame := range frames { + if matchesAny([]byte(frame), linuxStallAnchorFrames) { + for _, prefix := range []string{ + "__x64_", + "SYSC_", + "SyS_", + "compat_SYSC_", + "compat_SyS_", + } { + prev = strings.TrimPrefix(prev, prefix) + } + return prev, "" + } + prev = frame + } + return "", "did not find any anchor frame" +} + +var linuxStallAnchorFrames = []*regexp.Regexp{ + // Various generic functions that dispatch work. + // We also include some of their callers, so that if some names change + // we don't skip whole stacks and proceed parsing the next one. + compile("process_one_work"), // workqueue callback + compile("do_syscall_"), // syscall entry + compile("do_fast_syscall_"), // syscall entry + compile("netif_receive_skb"), // net receive entry point + compile("do_softirq"), + compile("call_timer_fn"), + compile("_run_timers"), + compile("run_timer_softirq"), + compile("run_ksoftirqd"), + compile("smpboot_thread_fn"), + compile("kthread"), + compile("start_secondary"), + compile("cpu_startup_entry"), + compile("ret_from_fork"), + // Important discriminated syscalls (file_operations callbacks, etc): + compile("vfs_write"), + compile("vfs_read"), + compile("vfs_iter_read"), + compile("vfs_iter_write"), + compile("do_iter_read"), + compile("do_iter_write"), + compile("vfs_ioctl"), + compile("blkdev_driver_ioctl"), + compile("blkdev_ioctl"), + compile("^call_read_iter"), + compile("^call_write_iter"), + compile("do_iter_readv_writev"), + compile("^call_mmap"), + compile("mmap_region"), + compile("do_mmap"), + compile("do_dentry_open"), + compile("vfs_open"), + // Socket operations: + compile("^sock_sendmsg"), + compile("^sock_recvmsg"), + compile("^sock_release"), + compile("^__sock_release"), + compile("__sys_setsockopt"), + compile("kernel_setsockopt"), + compile("sock_common_setsockopt"), + compile("__sys_listen"), + compile("kernel_listen"), + compile("sk_common_release"), + compile("^sock_mmap"), + compile("__sys_accept"), + compile("kernel_accept"), + compile("^sock_do_ioctl"), + compile("^sock_ioctl"), + compile("^compat_sock_ioctl"), + compile("^(compat_)?(SYSC|SyS|__sys|___sys|__do_sys|__se_sys|__x64_sys)_socketpair"), + compile("^(compat_)?(SYSC|SyS|__sys|___sys|__do_sys|__se_sys|__x64_sys)_connect"), + // Page fault entry points: + compile("__do_fault"), + compile("handle_mm_fault"), + compile("do_page_fault"), + compile("^page_fault$"), + // exit_to_usermode_loop callbacks: + compile("__fput"), + compile("task_work_run"), + compile("exit_to_usermode"), +} + var ( linuxSymbolizeRe = regexp.MustCompile(`(?:\[\<(?:[0-9a-f]+)\>\])?[ \t]+(?:[0-9]+:)?([a-zA-Z0-9_.]+)\+0x([0-9a-f]+)/0x([0-9a-f]+)`) stackFrameRe = regexp.MustCompile(`^ *(?:\[\<(?:[0-9a-f]+)\>\])?[ \t]+(?:[0-9]+:)?([a-zA-Z0-9_.]+)\+0x([0-9a-f]+)/0x([0-9a-f]+)`) @@ -697,6 +794,7 @@ var linuxOopses = []*oops{ compile("Call Trace:"), parseStackTrace, }, + extractor: linuxStallFrameExtractor, }, }, { @@ -963,18 +1061,8 @@ var linuxOopses = []*oops{ compile("apic_timer_interrupt"), parseStackTrace, }, - skip: []string{"apic_timer_interrupt", "rcu"}, - }, - }, - { - title: linuxRcuStall, - fmt: "INFO: rcu detected stall in %[1]v", - stack: &stackFmt{ - parts: []*regexp.Regexp{ - linuxRipFrame, - parseStackTrace, - }, - skip: []string{"apic_timer_interrupt", "rcu"}, + skip: []string{"apic_timer_interrupt", "rcu"}, + extractor: linuxStallFrameExtractor, }, }, { diff --git a/pkg/report/report.go b/pkg/report/report.go index ce953c8a1..27a71e937 100644 --- a/pkg/report/report.go +++ b/pkg/report/report.go @@ -234,8 +234,13 @@ type stackFmt struct { parts2 []*regexp.Regexp // Skip these functions in stack traces (matched as substring). skip []string + // Custom frame extractor (optional). + // Accepts set of all frames, returns guilty frame and corruption reason. + extractor frameExtractor } +type frameExtractor func(frames []string) (frame string, corrupted string) + var parseStackTrace *regexp.Regexp func compile(re string) *regexp.Regexp { @@ -367,24 +372,30 @@ func extractStackFrame(params *stackParams, stack *stackFmt, output []byte) (str if len(skip) != 0 { skipRe = regexp.MustCompile(strings.Join(skip, "|")) } - frame, corrupted := extractStackFrameImpl(params, output, skipRe, stack.parts) + extractor := stack.extractor + if extractor == nil { + extractor = func(frames []string) (string, string) { + return frames[0], "" + } + } + frame, corrupted := extractStackFrameImpl(params, output, skipRe, stack.parts, extractor) if frame != "" || len(stack.parts2) == 0 { return frame, corrupted } - return extractStackFrameImpl(params, output, skipRe, stack.parts2) + return extractStackFrameImpl(params, output, skipRe, stack.parts2, extractor) } func extractStackFrameImpl(params *stackParams, output []byte, skipRe *regexp.Regexp, - parts []*regexp.Regexp) (string, string) { - corrupted := "" + parts []*regexp.Regexp, extractor frameExtractor) (string, string) { s := bufio.NewScanner(bytes.NewReader(output)) + var frames []string nextPart: for _, part := range parts { if part == parseStackTrace { for s.Scan() { ln := bytes.Trim(s.Bytes(), "\r") - if corrupted == "" && matchesAny(ln, params.corruptedLines) { - corrupted = "corrupted line in report (1)" + if matchesAny(ln, params.corruptedLines) { + break nextPart } if matchesAny(ln, params.stackStartRes) { continue nextPart @@ -401,14 +412,14 @@ nextPart: } frame := ln[match[2]:match[3]] if skipRe == nil || !skipRe.Match(frame) { - return string(frame), corrupted + frames = append(frames, string(frame)) } } } else { for s.Scan() { ln := bytes.Trim(s.Bytes(), "\r") - if corrupted == "" && matchesAny(ln, params.corruptedLines) { - corrupted = "corrupted line in report (2)" + if matchesAny(ln, params.corruptedLines) { + break nextPart } match := part.FindSubmatchIndex(ln) if match == nil { @@ -417,14 +428,17 @@ nextPart: if len(match) == 4 && match[2] != -1 { frame := ln[match[2]:match[3]] if skipRe == nil || !skipRe.Match(frame) { - return string(frame), corrupted + frames = append(frames, string(frame)) } } break } } } - return "", corrupted + if len(frames) == 0 { + return "", "extracted no frames" + } + return extractor(frames) } func simpleLineParser(output []byte, oopses []*oops, params *stackParams, ignores []*regexp.Regexp) *Report { diff --git a/pkg/report/testdata/linux/report/145 b/pkg/report/testdata/linux/report/145 index 6a7348bf7..72f3df499 100644 --- a/pkg/report/testdata/linux/report/145 +++ b/pkg/report/testdata/linux/report/145 @@ -1,4 +1,4 @@ -TITLE: INFO: rcu detected stall in __xfrm_decode_session +TITLE: INFO: rcu detected stall in ipv6_rcv [ 323.830017] INFO: rcu_sched detected stalls on CPUs/tasks: [ 323.835676] (detected by 0, t=125007 jiffies, g=66299, c=66298, q=40) @@ -7,74 +7,15 @@ TITLE: INFO: rcu detected stall in __xfrm_decode_session [ 323.861597] Call Trace: [ 323.864155] <IRQ> [ 323.866280] sched_show_task+0x4a3/0x5e0 -[ 323.870319] ? can_nice.part.83+0x20/0x20 -[ 323.874437] ? vprintk_func+0x5e/0xc0 -[ 323.878207] ? printk+0xaa/0xca -[ 323.881459] ? show_regs_print_info+0x18/0x18 -[ 323.885931] ? show_regs_print_info+0x18/0x18 [ 323.890409] print_other_cpu_stall+0x996/0x1090 -[ 323.895047] ? print_irqtrace_events+0x270/0x270 -[ 323.899771] ? print_irqtrace_events+0x270/0x270 -[ 323.904510] ? rcu_stall_kick_kthreads.part.48+0x770/0x770 -[ 323.910106] ? check_noncircular+0x20/0x20 -[ 323.914309] ? print_irqtrace_events+0x270/0x270 -[ 323.919041] ? print_irqtrace_events+0x270/0x270 -[ 323.923766] ? __lock_acquire+0x664/0x3e00 -[ 323.927980] ? check_noncircular+0x20/0x20 -[ 323.932190] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 323.937352] ? __lock_acquire+0x664/0x3e00 -[ 323.941558] ? rcu_read_lock_sched_held+0x108/0x120 -[ 323.946544] ? update_cfs_rq_load_avg.part.69+0x23d/0x2d0 -[ 323.952048] ? attach_entity_load_avg+0x7a0/0x7a0 [ 323.956870] check_cpu_stall.isra.61+0x6e6/0x15b0 -[ 323.961682] ? __lock_acquire+0x664/0x3e00 -[ 323.965892] ? rcu_lockdep_current_cpu_online+0x190/0x190 -[ 323.971398] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 323.976563] ? __lock_acquire+0x664/0x3e00 -[ 323.980767] ? __lock_acquire+0x664/0x3e00 -[ 323.984984] ? check_noncircular+0x20/0x20 -[ 323.989196] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 323.994352] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 323.999518] ? lock_downgrade+0x980/0x980 -[ 324.003646] ? trace_hardirqs_off+0xd/0x10 -[ 324.007852] ? check_noncircular+0x20/0x20 -[ 324.012063] ? check_noncircular+0x20/0x20 -[ 324.016277] ? cpuacct_account_field+0x1e4/0x3b0 -[ 324.021015] ? check_noncircular+0x20/0x20 -[ 324.025220] ? find_held_lock+0x35/0x1d0 -[ 324.029251] ? __acct_update_integrals+0x339/0x4d0 -[ 324.034154] ? __lock_is_held+0xb6/0x140 [ 324.038202] rcu_check_callbacks+0x256/0xd00 -[ 324.042585] ? rcu_cpu_stall_reset+0x260/0x260 -[ 324.047139] ? trace_hardirqs_off+0xd/0x10 -[ 324.051344] ? raise_softirq+0x325/0x490 -[ 324.055381] ? raise_softirq_irqoff+0x320/0x320 -[ 324.060042] ? run_local_timers+0x18d/0x200 -[ 324.064331] ? timer_clear_idle+0x50/0x50 -[ 324.068446] ? account_process_tick+0xd4/0x3e0 -[ 324.072999] ? thread_group_cputime+0xef0/0xef0 -[ 324.077641] ? ktime_get_resolution_ns+0x300/0x300 [ 324.082553] update_process_times+0x30/0x60 [ 324.086846] tick_sched_handle+0x85/0x160 [ 324.090968] tick_sched_timer+0x42/0x120 [ 324.094999] __hrtimer_run_queues+0x358/0xe20 -[ 324.099472] ? tick_sched_do_timer+0xe0/0xe0 -[ 324.103855] ? hrtimer_fixup_init+0x70/0x70 -[ 324.108155] ? pvclock_read_flags+0x160/0x160 -[ 324.112624] ? lock_acquire+0x1d5/0x580 -[ 324.116570] ? kvm_clock_get_cycles+0x25/0x30 -[ 324.121034] ? ktime_get_update_offsets_now+0x34a/0x520 -[ 324.126371] ? do_timer+0x50/0x50 -[ 324.129800] ? do_raw_spin_trylock+0x190/0x190 [ 324.134365] hrtimer_interrupt+0x1c2/0x5e0 -[ 324.172031] ? apic_timer_interrupt+0xa9/0xb0 [ 324.138588] smp_apic_timer_interrupt+0x14a/0x700 -[ 324.143401] ? smp_call_function_single_interrupt+0x640/0x640 -[ 324.149252] ? smp_call_function_single_interrupt+0x640/0x640 -[ 324.155108] ? _raw_spin_unlock+0x22/0x30 -[ 324.159231] ? mark_held_locks+0xaf/0x100 -[ 324.163349] ? retint_kernel+0x10/0x10 -[ 324.167210] ? trace_hardirqs_off_thunk+0x1a/0x1c [ 324.172031] apic_timer_interrupt+0xa9/0xb0 [ 324.176323] RIP: 0010:__sanitizer_cov_trace_pc+0x0/0x50 [ 324.181650] RSP: 0018:ffff8801db206738 EFLAGS: 00000206 ORIG_RAX: ffffffffffffff11 @@ -83,193 +24,34 @@ TITLE: INFO: rcu detected stall in __xfrm_decode_session [ 324.203803] RBP: ffff8801db2067c8 R08: ffffed0038bea003 R09: ffffed0038bea003 [ 324.211046] R10: 000000000000000b R11: ffffed0038bea002 R12: ffff8801c5f40580 [ 324.218304] R13: ffff8801db206aa0 R14: 00000000000079d0 R15: dffffc0000000000 -[ 324.225565] ? _decode_session6+0x519/0x1380 -[ 324.229954] ? _decode_session6+0x761/0x1380 [ 324.234345] __xfrm_decode_session+0x68/0x110 [ 324.238814] __xfrm_policy_check+0x18c/0x2350 -[ 324.243279] ? save_stack+0xa3/0xd0 -[ 324.246876] ? save_stack+0x43/0xd0 -[ 324.250469] ? kasan_kmalloc+0xad/0xe0 -[ 324.254322] ? kasan_slab_alloc+0x12/0x20 -[ 324.258435] ? kmem_cache_alloc+0x12e/0x760 -[ 324.262723] ? raw6_local_deliver+0x737/0xa80 -[ 324.267185] ? ip6_input_finish+0x3c7/0x17a0 -[ 324.271564] ? ipv6_rcv+0xf37/0x1fa0 -[ 324.275249] ? __netif_receive_skb_core+0x1a41/0x3460 -[ 324.280418] ? __xfrm_route_forward+0x670/0x670 -[ 324.285054] ? ip6_finish_output2+0xba0/0x23a0 -[ 324.289605] ? ip6_finish_output+0x6bb/0xaf0 -[ 324.293980] ? ip6_output+0x1eb/0x840 -[ 324.297748] ? ip6_local_out+0x95/0x160 -[ 324.301692] ? check_noncircular+0x20/0x20 -[ 324.305895] ? inet_sendmsg+0x11f/0x5e0 -[ 324.309836] ? sock_sendmsg+0xca/0x110 -[ 324.313694] ? SYSC_sendto+0x361/0x5c0 -[ 324.317547] ? SyS_sendto+0x40/0x50 -[ 324.321142] ? entry_SYSCALL_64_fastpath+0x23/0x9a -[ 324.326041] ? SyS_sendto+0x40/0x50 -[ 324.329636] ? entry_SYSCALL_64_fastpath+0x23/0x9a -[ 324.334536] ? free_obj_work+0x690/0x690 -[ 324.338567] ? debug_object_free+0x5a0/0x5a0 -[ 324.342948] ? memcpy+0x45/0x50 -[ 324.346199] ? __copy_skb_header+0x31f/0x540 -[ 324.350576] ? skb_may_tx_timestamp.part.62+0xc0/0xc0 -[ 324.355736] ? __lock_is_held+0xb6/0x140 -[ 324.359775] ? __skb_clone+0x2ac/0xa70 [ 324.363656] rawv6_rcv+0x8f6/0x1200 -[ 324.367262] ? raw6_icmp_error+0x740/0x740 -[ 324.371463] ? __raw_v6_lookup+0x29f/0x390 [ 324.375676] raw6_local_deliver+0x819/0xa80 -[ 324.379977] ? rawv6_rcv+0x1200/0x1200 -[ 324.383833] ? ip6_rcv_finish+0x7a0/0x7a0 -[ 324.387950] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 324.393106] ? lock_downgrade+0x980/0x980 -[ 324.397227] ? lock_release+0xa40/0xa40 -[ 324.401174] ? __lock_is_held+0xb6/0x140 [ 324.405216] ip6_input_finish+0x3c7/0x17a0 -[ 324.409418] ? ip6_input+0x3b4/0x560 -[ 324.413102] ? lock_downgrade+0x980/0x980 -[ 324.417231] ? ip6_rcv_finish+0x7a0/0x7a0 -[ 324.421351] ? nf_hook_slow+0xd3/0x1a0 [ 324.425219] ip6_input+0xe9/0x560 -[ 324.428645] ? ip6_input_finish+0x17a0/0x17a0 -[ 324.433114] ? find_held_lock+0x35/0x1d0 -[ 324.437149] ? ip6_rcv_finish+0x7a0/0x7a0 -[ 324.441267] ? ipv6_rcv+0x16cd/0x1fa0 [ 324.445039] ip6_rcv_finish+0x1a9/0x7a0 -[ 324.448985] ? ip6_make_skb+0x580/0x580 -[ 324.452931] ? __lock_is_held+0xb6/0x140 -[ 324.456967] ? nf_hook_slow+0xd3/0x1a0 [ 324.460829] ipv6_rcv+0xf37/0x1fa0 -[ 324.464348] ? ip6_input+0x560/0x560 -[ 324.468032] ? print_irqtrace_events+0x270/0x270 -[ 324.472762] ? rb_insert_color+0x1580/0x1580 -[ 324.477144] ? check_noncircular+0x20/0x20 -[ 324.481344] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 324.486500] ? __lock_acquire+0x664/0x3e00 -[ 324.490717] ? ip6_make_skb+0x580/0x580 -[ 324.494665] ? ip6_input+0x560/0x560 [ 324.498351] __netif_receive_skb_core+0x1a41/0x3460 -[ 324.503336] ? find_held_lock+0x35/0x1d0 -[ 324.507375] ? nf_ingress+0x9f0/0x9f0 -[ 324.511145] ? lock_downgrade+0x980/0x980 -[ 324.515285] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 324.520456] ? update_curr+0x47b/0xa60 -[ 324.524323] ? __lock_acquire+0x664/0x3e00 -[ 324.528527] ? numa_migrate_preferred+0x250/0x250 -[ 324.533347] ? _find_next_bit+0xee/0x120 -[ 324.537386] ? account_entity_enqueue+0x3c8/0x6e0 -[ 324.542205] ? check_noncircular+0x20/0x20 -[ 324.546411] ? check_noncircular+0x20/0x20 -[ 324.550619] ? enqueue_task_fair+0x3b7/0x2950 -[ 324.555083] ? find_held_lock+0x35/0x1d0 -[ 324.559118] ? check_preempt_wakeup+0xb20/0xb20 -[ 324.563765] ? find_held_lock+0x35/0x1d0 -[ 324.567803] ? lock_acquire+0x1d5/0x580 -[ 324.571756] ? process_backlog+0x45f/0x740 -[ 324.575958] ? lock_acquire+0x1d5/0x580 -[ 324.579901] ? process_backlog+0x1ab/0x740 -[ 324.584111] ? lock_release+0xa40/0xa40 [ 324.588071] __netif_receive_skb+0x2c/0x1b0 -[ 324.592361] ? __netif_receive_skb+0x2c/0x1b0 [ 324.596827] process_backlog+0x203/0x740 -[ 324.600854] ? mark_held_locks+0xaf/0x100 [ 324.604982] net_rx_action+0x792/0x1910 -[ 324.608940] ? napi_complete_done+0x6c0/0x6c0 -[ 324.613407] ? trace_hardirqs_off+0xd/0x10 -[ 324.617611] ? _raw_spin_unlock_irqrestore+0xa6/0xba -[ 324.622684] ? try_to_wake_up+0xf9/0x1600 -[ 324.626801] ? lock_downgrade+0x980/0x980 -[ 324.630925] ? migrate_swap_stop+0x970/0x970 -[ 324.635304] ? check_noncircular+0x20/0x20 -[ 324.639512] ? check_noncircular+0x20/0x20 -[ 324.643721] ? lock_acquire+0x1d5/0x580 -[ 324.647665] ? lock_acquire+0x1d5/0x580 -[ 324.651610] ? __hrtimer_run_queues+0x394/0xe20 -[ 324.656247] ? lock_downgrade+0x980/0x980 -[ 324.660367] ? lock_release+0xa40/0xa40 -[ 324.664322] ? find_held_lock+0x35/0x1d0 -[ 324.668363] ? clockevents_program_event+0x163/0x2e0 -[ 324.673435] ? lock_downgrade+0x980/0x980 -[ 324.677564] ? pvclock_read_flags+0x160/0x160 -[ 324.682041] ? check_noncircular+0x20/0x20 -[ 324.686245] ? print_irqtrace_events+0x270/0x270 -[ 324.690972] ? ktime_get_resolution_ns+0x300/0x300 -[ 324.695866] ? lock_downgrade+0x980/0x980 -[ 324.699984] ? check_noncircular+0x20/0x20 -[ 324.704187] ? do_timer+0x50/0x50 -[ 324.707615] ? __lock_is_held+0xb6/0x140 [ 324.711663] __do_softirq+0x2d7/0xb85 -[ 324.715432] ? task_prio+0x40/0x40 -[ 324.718949] ? __irqentry_text_end+0x4/0x4 -[ 324.723152] ? irq_exit+0xbb/0x200 -[ 324.726660] ? smp_apic_timer_interrupt+0x16b/0x700 -[ 324.731646] ? smp_reschedule_interrupt+0xe6/0x670 -[ 324.736543] ? smp_call_function_single_interrupt+0x640/0x640 -[ 324.742400] ? _raw_spin_lock+0x32/0x40 -[ 324.746356] ? _raw_spin_unlock+0x22/0x30 -[ 324.750474] ? handle_edge_irq+0x2b4/0x7c0 -[ 324.754681] ? task_prio+0x40/0x40 -[ 324.758199] ? trace_hardirqs_off_thunk+0x1a/0x1c [ 324.763025] do_softirq_own_stack+0x2a/0x40 [ 324.767312] </IRQ> [ 324.769522] do_softirq.part.21+0x14d/0x190 -[ 324.773813] ? ip6_finish_output2+0xb6d/0x23a0 [ 324.778364] __local_bh_enable_ip+0x1ee/0x230 [ 324.782830] ip6_finish_output2+0xba0/0x23a0 -[ 324.787213] ? print_irqtrace_events+0x270/0x270 -[ 324.791951] ? ip6_sk_dst_lookup_flow+0x7f0/0x7f0 -[ 324.796764] ? prandom_u32+0x24/0x30 -[ 324.800447] ? ip_idents_reserve+0x1e6/0x2a0 -[ 324.804828] ? ipv4_sysctl_rtcache_flush+0x70/0x70 -[ 324.809732] ? __lock_is_held+0xb6/0x140 -[ 324.813774] ? rcu_read_lock_sched_held+0x108/0x120 -[ 324.818758] ? ip6_copy_metadata+0x62a/0xad0 -[ 324.823132] ? __kmalloc_track_caller+0x46a/0x760 -[ 324.827944] ? dst_output+0x140/0x140 -[ 324.831715] ? memcpy+0x45/0x50 [ 324.834968] ip6_fragment+0x25f2/0x3470 -[ 324.838912] ? ip6_fragment+0x25f2/0x3470 -[ 324.843043] ? ip6_sk_dst_lookup_flow+0x7f0/0x7f0 -[ 324.847864] ? ip6_forward+0x33d0/0x33d0 -[ 324.851902] ? ip6_mtu+0x1c7/0x4d0 -[ 324.855410] ? ip6_dst_ifdown+0x3d0/0x3d0 [ 324.859537] ip6_finish_output+0x6bb/0xaf0 [ 324.863753] ip6_output+0x1eb/0x840 -[ 324.867362] ? ip6_finish_output+0xaf0/0xaf0 -[ 324.871740] ? lock_acquire+0x1d5/0x580 -[ 324.875683] ? rawv6_sendmsg+0x1d74/0x3e70 -[ 324.879892] ? ip6_fragment+0x3470/0x3470 [ 324.884025] ip6_local_out+0x95/0x160 [ 324.887802] ip6_send_skb+0xa1/0x330 [ 324.891487] ip6_push_pending_frames+0xb3/0xe0 [ 324.896041] rawv6_sendmsg+0x2ee9/0x3e70 -[ 324.900090] ? rawv6_bind+0x8c0/0x8c0 -[ 324.903870] ? avc_has_perm+0x35e/0x680 -[ 324.907814] ? lock_downgrade+0x980/0x980 -[ 324.911934] ? lock_release+0xa40/0xa40 -[ 324.915876] ? __handle_mm_fault+0x3ce0/0x3ce0 -[ 324.920436] ? up_read+0x1a/0x40 -[ 324.923776] ? avc_has_perm+0x43e/0x680 -[ 324.927724] ? avc_has_perm_noaudit+0x520/0x520 -[ 324.932372] ? do_page_fault+0xee/0x720 -[ 324.936313] ? iterate_fd+0x3f0/0x3f0 -[ 324.940091] ? find_held_lock+0x35/0x1d0 [ 324.944155] inet_sendmsg+0x11f/0x5e0 -[ 324.947922] ? inet_sendmsg+0x11f/0x5e0 -[ 324.951869] ? inet_recvmsg+0x5f0/0x5f0 -[ 324.955816] ? selinux_socket_sendmsg+0x36/0x40 -[ 324.960454] ? security_socket_sendmsg+0x89/0xb0 -[ 324.965181] ? inet_recvmsg+0x5f0/0x5f0 [ 324.969127] sock_sendmsg+0xca/0x110 [ 324.972812] SYSC_sendto+0x361/0x5c0 -[ 324.976502] ? SYSC_connect+0x4a0/0x4a0 -[ 324.980454] ? __might_sleep+0x95/0x190 -[ 324.984399] ? exit_to_usermode_loop+0x8c/0x310 -[ 324.989049] ? trace_event_raw_event_sys_exit+0x260/0x260 -[ 324.994566] ? syscall_return_slowpath+0x2ad/0x550 -[ 324.999464] ? prepare_exit_to_usermode+0x340/0x340 -[ 325.004450] ? entry_SYSCALL_64_fastpath+0x5/0x9a [ 325.009270] SyS_sendto+0x40/0x50 [ 325.012699] entry_SYSCALL_64_fastpath+0x23/0x9a [ 325.017423] RIP: 0033:0x452ac9 diff --git a/pkg/report/testdata/linux/report/171 b/pkg/report/testdata/linux/report/171 index cceb2a3f0..089735f0d 100644 --- a/pkg/report/testdata/linux/report/171 +++ b/pkg/report/testdata/linux/report/171 @@ -1,4 +1,4 @@ -TITLE: INFO: rcu detected stall in mulaw_decode +TITLE: INFO: rcu detected stall in snd_pcm_oss_write [ 196.114026] INFO: rcu_sched detected stalls on CPUs/tasks: [ 196.119690] (detected by 0, t=125002 jiffies, g=15654, c=15653, q=133) @@ -7,75 +7,15 @@ TITLE: INFO: rcu detected stall in mulaw_decode [ 196.145712] Call Trace: [ 196.148271] <IRQ> [ 196.150403] sched_show_task+0x4a3/0x5e0 -[ 196.154447] ? can_nice.part.83+0x20/0x20 -[ 196.158577] ? vprintk_func+0x5e/0xc0 -[ 196.162354] ? printk+0xaa/0xca -[ 196.165610] ? show_regs_print_info+0x18/0x18 -[ 196.170080] ? show_regs_print_info+0x18/0x18 [ 196.174578] print_other_cpu_stall+0x996/0x1090 -[ 196.179228] ? __lock_is_held+0xb6/0x140 -[ 196.183282] ? rcu_stall_kick_kthreads.part.48+0x770/0x770 -[ 196.188879] ? check_noncircular+0x20/0x20 -[ 196.193092] ? print_irqtrace_events+0x270/0x270 -[ 196.197837] ? print_irqtrace_events+0x270/0x270 -[ 196.202579] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 196.207744] ? check_noncircular+0x20/0x20 -[ 196.211962] ? print_irqtrace_events+0x270/0x270 -[ 196.216697] ? print_irqtrace_events+0x270/0x270 -[ 196.221440] ? __lock_acquire+0x664/0x3e00 -[ 196.225662] ? rcu_read_lock_sched_held+0x108/0x120 -[ 196.230657] ? update_cfs_rq_load_avg.part.69+0x23d/0x2d0 [ 196.236196] check_cpu_stall.isra.61+0x6e6/0x15b0 -[ 196.241015] ? check_noncircular+0x20/0x20 -[ 196.245241] ? rcu_lockdep_current_cpu_online+0x190/0x190 -[ 196.250756] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 196.255942] ? find_held_lock+0x35/0x1d0 -[ 196.259996] ? check_noncircular+0x20/0x20 -[ 196.264213] ? lock_downgrade+0x980/0x980 -[ 196.268340] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 196.273513] ? lock_release+0xa40/0xa40 -[ 196.277475] ? check_noncircular+0x20/0x20 -[ 196.281699] ? check_noncircular+0x20/0x20 -[ 196.285922] ? cpuacct_account_field+0x1e4/0x3b0 -[ 196.290662] ? check_noncircular+0x20/0x20 -[ 196.294870] ? find_held_lock+0x35/0x1d0 -[ 196.298911] ? __acct_update_integrals+0x339/0x4d0 -[ 196.303825] ? __lock_is_held+0xb6/0x140 [ 196.307884] rcu_check_callbacks+0x256/0xd00 -[ 196.312279] ? rcu_cpu_stall_reset+0x260/0x260 -[ 196.316843] ? trace_hardirqs_off+0xd/0x10 -[ 196.321054] ? raise_softirq+0x325/0x490 -[ 196.325095] ? raise_softirq_irqoff+0x320/0x320 -[ 196.329745] ? read_boot_clock64+0x70/0x70 -[ 196.333961] ? lock_downgrade+0x980/0x980 -[ 196.338085] ? lock_downgrade+0x980/0x980 -[ 196.342218] ? run_local_timers+0x18d/0x200 -[ 196.346515] ? timer_clear_idle+0x50/0x50 -[ 196.350644] ? account_process_tick+0xd4/0x3e0 -[ 196.355210] ? thread_group_cputime+0xef0/0xef0 -[ 196.359859] ? ktime_get_resolution_ns+0x300/0x300 [ 196.364779] update_process_times+0x30/0x60 [ 196.369080] tick_sched_handle+0x85/0x160 [ 196.373206] tick_sched_timer+0x42/0x120 [ 196.377249] __hrtimer_run_queues+0x358/0xe20 -[ 196.381733] ? tick_sched_do_timer+0xe0/0xe0 -[ 196.386122] ? hrtimer_fixup_init+0x70/0x70 -[ 196.390435] ? pvclock_read_flags+0x160/0x160 -[ 196.394924] ? kvm_clock_get_cycles+0x25/0x30 -[ 196.399397] ? ktime_get_update_offsets_now+0x34a/0x520 -[ 196.404747] ? do_timer+0x50/0x50 -[ 196.408179] ? native_apic_msr_write+0x5c/0x80 -[ 196.412739] ? do_raw_spin_trylock+0x190/0x190 -[ 196.417301] ? lapic_next_event+0x54/0x80 -[ 196.421436] ? clockevents_program_event+0x108/0x2e0 [ 196.426526] hrtimer_interrupt+0x1c2/0x5e0 [ 196.430767] smp_apic_timer_interrupt+0x14a/0x700 -[ 196.435592] ? smp_call_function_single_interrupt+0x640/0x640 -[ 196.441455] ? _raw_spin_lock+0x32/0x40 -[ 196.445410] ? _raw_spin_unlock+0x22/0x30 -[ 196.449533] ? handle_edge_irq+0x2b4/0x7c0 -[ 196.453747] ? task_prio+0x40/0x40 -[ 196.457279] ? trace_hardirqs_off_thunk+0x1a/0x1c [ 196.462113] apic_timer_interrupt+0xa9/0xb0 [ 196.466405] </IRQ> [ 196.468619] RIP: 0010:check_memory_region+0x38/0x190 @@ -85,41 +25,15 @@ TITLE: INFO: rcu detected stall in mulaw_decode [ 196.495875] RBP: ffff8801bd2ff878 R08: ffffc9000160020a R09: dffffc0000000000 [ 196.503120] R10: 0000000000000001 R11: ffffed0037a5ff2e R12: ffffc9000160020a [ 196.510365] R13: ffff8801bd2ff970 R14: dffffc0000000000 R15: ffffc9000160020a -[ 196.517632] ? mulaw_decode+0x52f/0x770 -[ 196.521592] ? check_noncircular+0x20/0x20 [ 196.525806] memcpy+0x37/0x50 [ 196.528895] mulaw_decode+0x52f/0x770 -[ 196.532706] ? snd_pcm_plugin_build_linear+0x8a0/0x8a0 -[ 196.537960] ? snd_pcm_plugin_client_channels+0x17/0x80 -[ 196.543305] ? io_src_channels+0x10a/0x210 [ 196.547525] mulaw_transfer+0x222/0x270 -[ 196.551473] ? lock_release+0xa40/0xa40 -[ 196.555431] ? mulaw_encode+0x7c0/0x7c0 [ 196.559384] snd_pcm_plug_write_transfer+0x22d/0x420 -[ 196.564475] ? snd_pcm_plug_client_channels_buf+0x3f0/0x3f0 -[ 196.570164] ? snd_pcm_format_physical_width+0x5b/0x70 -[ 196.575419] ? snd_pcm_plug_client_channels_buf+0x1db/0x3f0 [ 196.581116] snd_pcm_oss_write2+0x260/0x420 -[ 196.585425] ? snd_pcm_oss_write3+0x1b0/0x1b0 [ 196.589913] snd_pcm_oss_write+0x5fe/0x830 -[ 196.594157] ? snd_pcm_oss_ioctl_compat+0x30/0x30 -[ 196.598991] ? snd_pcm_oss_ioctl_compat+0x30/0x30 [ 196.603813] __vfs_write+0xef/0x970 -[ 196.607418] ? rcu_note_context_switch+0x710/0x710 -[ 196.612327] ? kernel_read+0x120/0x120 -[ 196.616193] ? __might_sleep+0x95/0x190 -[ 196.620146] ? _cond_resched+0x14/0x30 -[ 196.624010] ? __inode_security_revalidate+0xd9/0x130 -[ 196.629180] ? avc_policy_seqno+0x9/0x20 -[ 196.633218] ? selinux_file_permission+0x82/0x460 -[ 196.638045] ? security_file_permission+0x89/0x1e0 -[ 196.642956] ? rw_verify_area+0xe5/0x2b0 -[ 196.646991] ? __fdget_raw+0x20/0x20 [ 196.650693] vfs_write+0x189/0x510 [ 196.654218] SyS_write+0xef/0x220 -[ 196.657656] ? SyS_read+0x220/0x220 -[ 196.661258] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 196.666256] ? trace_hardirqs_on_thunk+0x1a/0x1c [ 196.670999] entry_SYSCALL_64_fastpath+0x23/0x9a [ 196.675728] RIP: 0033:0x452ac9 [ 196.678895] RSP: 002b:00007fa354a13c58 EFLAGS: 00000212 ORIG_RAX: 0000000000000001 diff --git a/pkg/report/testdata/linux/report/200 b/pkg/report/testdata/linux/report/200 index d6c1f9deb..d8e149eaf 100644 --- a/pkg/report/testdata/linux/report/200 +++ b/pkg/report/testdata/linux/report/200 @@ -1,4 +1,4 @@ -TITLE: general protection fault in ip6t_do_table +TITLE: general protection fault in corrupted CORRUPTED: Y [ 73.452724] FAULT_INJECTION: forcing a failure. diff --git a/pkg/report/testdata/linux/report/207 b/pkg/report/testdata/linux/report/207 index 055c28d04..168b69049 100644 --- a/pkg/report/testdata/linux/report/207 +++ b/pkg/report/testdata/linux/report/207 @@ -1,4 +1,4 @@ -TITLE: INFO: rcu detected stall in free_percpu +TITLE: INFO: rcu detected stall in bpf_map_free_deferred isob2 login: [ 709.689951] INFO: rcu_sched detected stalls on CPUs/tasks: [ 709.695737] (detected by 0, t=125002 jiffies, g=6134, c=6133, q=11) diff --git a/pkg/report/testdata/linux/report/235 b/pkg/report/testdata/linux/report/235 index 2b6228624..08599850f 100644 --- a/pkg/report/testdata/linux/report/235 +++ b/pkg/report/testdata/linux/report/235 @@ -1,4 +1,4 @@ -TITLE: INFO: task hung in should_fail +TITLE: INFO: task hung in corrupted CORRUPTED: Y [ 246.707981] FAULT_INJECTION: forcing a failure. diff --git a/pkg/report/testdata/linux/report/244 b/pkg/report/testdata/linux/report/244 index e8b1b55ce..ba049d0ba 100644 --- a/pkg/report/testdata/linux/report/244 +++ b/pkg/report/testdata/linux/report/244 @@ -1,4 +1,4 @@ -TITLE: WARNING: suspicious RCU usage in should_fail +TITLE: WARNING: suspicious RCU usage in corrupted CORRUPTED: Y [ 184.760966] ============================= diff --git a/pkg/report/testdata/linux/report/263 b/pkg/report/testdata/linux/report/263 new file mode 100644 index 000000000..032cc08d0 --- /dev/null +++ b/pkg/report/testdata/linux/report/263 @@ -0,0 +1,48 @@ +TITLE: INFO: rcu detected stall in snd_seq_write + +[ 599.875361] rcu: INFO: rcu_sched self-detected stall on CPU +[ 599.881199] rcu: 0-....: (105000 ticks this GP) idle=2f6/1/0x4000000000000002 softirq=23001/23001 fqs=26239 +[ 599.891361] rcu: (t=105008 jiffies g=49145 q=2382) +[ 599.896509] NMI backtrace for cpu 0 +[ 599.900150] CPU: 0 PID: 8551 Comm: syz-executor7 Not tainted 4.18.0-next-20180824+ #47 +[ 599.908198] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 599.917548] Call Trace: +[ 599.920132] <IRQ> +[ 599.922286] dump_stack+0x1c9/0x2b4 +[ 599.944682] nmi_cpu_backtrace.cold.3+0x48/0x88 +[ 599.954558] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 599.959858] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 599.965052] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 599.974486] print_cpu_stall.cold.78+0x2fb/0x59c +[ 599.995787] rcu_check_callbacks+0xd4a/0x15a0 +[ 600.093087] update_process_times+0x2d/0x70 +[ 600.097415] tick_sched_handle+0x9f/0x180 +[ 600.101564] tick_sched_timer+0x45/0x130 +[ 600.105627] __hrtimer_run_queues+0x3eb/0xff0 +[ 600.150990] hrtimer_interrupt+0x2f3/0x750 +[ 600.155241] smp_apic_timer_interrupt+0x16d/0x6a0 +[ 600.207857] apic_timer_interrupt+0xf/0x20 +[ 600.212089] </IRQ> +[ 600.214345] RIP: 0010:lock_release+0xa2/0x9f0 +[ 600.237744] RSP: 0018:ffff8801aa0078c8 EFLAGS: 00000292 ORIG_RAX: ffffffffffffff13 +[ 600.245455] RAX: 2ec5cd2493d34200 RBX: 1ffff10035400f1e RCX: 0000000000000000 +[ 600.252724] RDX: dffffc0000000000 RSI: 0000000000000000 RDI: ffff8801a9b56bbc +[ 600.259994] RBP: ffff8801aa0079f8 R08: 00000000000010df R09: 0000000000000001 +[ 600.267263] R10: ffff8801a9b56be8 R11: 737c3e5c87308161 R12: ffff8801aa0079d0 +[ 600.274530] R13: ffff8801c778d4c8 R14: ffff8801aa007ae0 R15: ffff8801a9b56380 +[ 600.307088] __might_fault+0x19e/0x1e0 +[ 600.310986] _copy_from_user+0x30/0x150 +[ 600.315053] snd_seq_write+0x472/0x8d0 +[ 600.332960] __vfs_write+0x117/0x9d0 +[ 600.379161] vfs_write+0x1fc/0x560 +[ 600.382717] ksys_write+0x101/0x260 +[ 600.390441] __x64_sys_write+0x73/0xb0 +[ 600.398306] do_syscall_64+0x1b9/0x820 +[ 600.436382] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 600.441569] RIP: 0033:0x457089 +[ 600.463760] RSP: 002b:00007f09af65dc78 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 +[ 600.471476] RAX: ffffffffffffffda RBX: 00007f09af65e6d4 RCX: 0000000000457089 +[ 600.478757] RDX: 00000000ffffff76 RSI: 0000000020000000 RDI: 0000000000000003 +[ 600.486032] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 600.493302] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 600.500573] R13: 00000000004d78a8 R14: 00000000004ca886 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/264 b/pkg/report/testdata/linux/report/264 new file mode 100644 index 000000000..0ca42fafe --- /dev/null +++ b/pkg/report/testdata/linux/report/264 @@ -0,0 +1,105 @@ +TITLE: BUG: soft lockup in sys_rmdir + +[ 281.646907] watchdog: BUG: soft lockup - CPU#1 stuck for 134s! [syz-executor4:4588] +[ 281.654759] Modules linked in: +[ 281.657946] irq event stamp: 76609500 +[ 281.661752] hardirqs last enabled at (76609499): [<ffffffff81c66ccf>] d_walk+0x80f/0xc80 +[ 281.670066] hardirqs last disabled at (76609500): [<ffffffff87800905>] interrupt_entry+0xb5/0xf0 +[ 281.679024] softirqs last enabled at (468244): [<ffffffff87a00778>] __do_softirq+0x778/0xaf5 +[ 281.687696] softirqs last disabled at (468177): [<ffffffff814762c1>] irq_exit+0x1d1/0x200 +[ 281.696008] CPU: 1 PID: 4588 Comm: syz-executor4 Not tainted 4.17.0-rc7+ #81 +[ 281.703881] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 281.713233] RIP: 0010:d_walk+0x840/0xc80 +[ 281.717286] RSP: 0018:ffff880183f5f9f8 EFLAGS: 00000293 ORIG_RAX: ffffffffffffff13 +[ 281.724995] RAX: ffff8801b0d3c680 RBX: 0000000000000293 RCX: 1ffff100361a79e5 +[ 281.732263] RDX: 0000000000000000 RSI: ffffffff81c66cfb RDI: 0000000000000293 +[ 281.739552] RBP: ffff880183f5fb78 R08: ffff8801b0d3ceb8 R09: 0000000000000006 +[ 281.746815] R10: ffff8801b0d3c680 R11: 0000000000000000 R12: 0000000000000200 +[ 281.754080] R13: dffffc0000000000 R14: 1ffff100307ebf82 R15: ffff880183f5fb50 +[ 281.761359] FS: 000000000217c940(0000) GS:ffff8801daf00000(0000) knlGS:0000000000000000 +[ 281.769577] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 281.775450] CR2: 00007ffca8309eac CR3: 0000000183eb2000 CR4: 00000000001406e0 +[ 281.782717] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 281.789980] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 281.797240] Call Trace: +[ 281.829726] shrink_dcache_parent+0x179/0x230 +[ 281.859894] vfs_rmdir+0x202/0x470 +[ 281.863431] do_rmdir+0x523/0x610 +[ 281.890027] __x64_sys_rmdir+0x36/0x40 +[ 281.893915] do_syscall_64+0x1b1/0x800 +[ 281.922815] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 281.927998] RIP: 0033:0x455777 +[ 281.931184] RSP: 002b:00007ffca830a5b8 EFLAGS: 00000206 ORIG_RAX: 0000000000000054 +[ 281.938889] RAX: ffffffffffffffda RBX: 0000000000000065 RCX: 0000000000455777 +[ 281.946179] RDX: 0000000000000000 RSI: 00007ffca830c360 RDI: 00007ffca830c360 +[ 281.953444] RBP: 00007ffca830c360 R08: 0000000000000000 R09: 0000000000000001 +[ 281.960792] R10: 0000000000000006 R11: 0000000000000206 R12: 000000000217d940 +[ 281.968057] R13: 0000000000000000 R14: 0000000000000198 R15: 000000000001f12d +[ 281.994904] Sending NMI from CPU 1 to CPUs 0: +[ 281.999398] NMI backtrace for cpu 0 +[ 281.999408] CPU: 0 PID: 4581 Comm: syz-executor6 Not tainted 4.17.0-rc7+ #81 +[ 281.999413] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 281.999428] RIP: 0010:select_collect+0x249/0x5f0 +[ 281.999433] RSP: 0018:ffff880189947908 EFLAGS: 00000286 +[ 281.999440] RAX: 0000000000000000 RBX: ffffed0031328f24 RCX: ffffffff81c5c3b4 +[ 281.999445] RDX: 0000000000000000 RSI: ffffffff81c5c4fa RDI: ffff880189947c28 +[ 281.999450] RBP: ffff8801899479e8 R08: ffff88018993e5c0 R09: ffffed0039dde7a4 +[ 281.999455] R10: ffffed0039dde7a4 R11: ffff8801ceef3d23 R12: 0000000000000000 +[ 281.999461] R13: 1ffff10031328f28 R14: ffff880189947c18 R15: dffffc0000000000 +[ 281.999468] FS: 0000000001790940(0000) GS:ffff8801dae00000(0000) knlGS:0000000000000000 +[ 281.999474] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 281.999479] CR2: 00007ffebea69e9c CR3: 000000018989f000 CR4: 00000000001406f0 +[ 281.999487] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 281.999491] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 281.999493] Call Trace: +[ 281.999541] d_walk+0x3c3/0xc80 +[ 281.999601] shrink_dcache_parent+0x179/0x230 +[ 281.999670] vfs_rmdir+0x202/0x470 +[ 281.999679] do_rmdir+0x523/0x610 +[ 281.999740] __x64_sys_rmdir+0x36/0x40 +[ 281.999748] do_syscall_64+0x1b1/0x800 +[ 281.999806] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 281.999812] RIP: 0033:0x455777 +[ 281.999816] RSP: 002b:00007ffebea6a5a8 EFLAGS: 00000202 ORIG_RAX: 0000000000000054 +[ 281.999824] RAX: ffffffffffffffda RBX: 0000000000000065 RCX: 0000000000455777 +[ 281.999829] RDX: 0000000000000000 RSI: 00007ffebea6c350 RDI: 00007ffebea6c350 +[ 281.999833] RBP: 00007ffebea6c350 R08: 0000000000000000 R09: 0000000000000001 +[ 281.999838] R10: 0000000000000006 R11: 0000000000000202 R12: 0000000001791940 +[ 281.999843] R13: 0000000000000000 R14: 0000000000000192 R15: 000000000001f12b +[ 282.000407] Kernel panic - not syncing: softlockup: hung tasks +[ 282.316419] CPU: 1 PID: 4588 Comm: syz-executor4 Tainted: G L 4.17.0-rc7+ #81 +[ 282.324992] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 282.334334] Call Trace: +[ 282.336997] <IRQ> +[ 282.339152] dump_stack+0x1b9/0x294 +[ 282.352760] panic+0x22f/0x4de +[ 282.375494] watchdog_timer_fn.cold.5+0xf3/0x126 +[ 282.380251] __hrtimer_run_queues+0x3e3/0x10a0 +[ 282.432572] hrtimer_interrupt+0x2f3/0x750 +[ 282.436818] smp_apic_timer_interrupt+0x15d/0x710 +[ 282.468292] apic_timer_interrupt+0xf/0x20 +[ 282.472527] </IRQ> +[ 282.474759] RIP: 0010:d_walk+0x840/0xc80 +[ 282.478811] RSP: 0018:ffff880183f5f9f8 EFLAGS: 00000293 ORIG_RAX: ffffffffffffff13 +[ 282.486516] RAX: ffff8801b0d3c680 RBX: 0000000000000293 RCX: 1ffff100361a79e5 +[ 282.493778] RDX: 0000000000000000 RSI: ffffffff81c66cfb RDI: 0000000000000293 +[ 282.501043] RBP: ffff880183f5fb78 R08: ffff8801b0d3ceb8 R09: 0000000000000006 +[ 282.508310] R10: ffff8801b0d3c680 R11: 0000000000000000 R12: 0000000000000200 +[ 282.515587] R13: dffffc0000000000 R14: 1ffff100307ebf82 R15: ffff880183f5fb50 +[ 282.556091] shrink_dcache_parent+0x179/0x230 +[ 282.586289] vfs_rmdir+0x202/0x470 +[ 282.589830] do_rmdir+0x523/0x610 +[ 282.616379] __x64_sys_rmdir+0x36/0x40 +[ 282.620264] do_syscall_64+0x1b1/0x800 +[ 282.649064] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 282.654251] RIP: 0033:0x455777 +[ 282.657434] RSP: 002b:00007ffca830a5b8 EFLAGS: 00000206 ORIG_RAX: 0000000000000054 +[ 282.665140] RAX: ffffffffffffffda RBX: 0000000000000065 RCX: 0000000000455777 +[ 282.672404] RDX: 0000000000000000 RSI: 00007ffca830c360 RDI: 00007ffca830c360 +[ 282.679668] RBP: 00007ffca830c360 R08: 0000000000000000 R09: 0000000000000001 +[ 282.687015] R10: 0000000000000006 R11: 0000000000000206 R12: 000000000217d940 +[ 282.694282] R13: 0000000000000000 R14: 0000000000000198 R15: 000000000001f12d +[ 282.702335] Dumping ftrace buffer: +[ 282.706164] (ftrace buffer empty) +[ 282.709862] Kernel Offset: disabled +[ 282.713505] Rebooting in 86400 seconds.. diff --git a/pkg/report/testdata/linux/report/265 b/pkg/report/testdata/linux/report/265 new file mode 100644 index 000000000..a21cdac90 --- /dev/null +++ b/pkg/report/testdata/linux/report/265 @@ -0,0 +1,150 @@ +TITLE: BUG: soft lockup in snd_rawmidi_write + +[ 352.504666] watchdog: BUG: soft lockup - CPU#1 stuck for 134s! [syz-executor2:10431] +[ 352.512610] Modules linked in: +[ 352.515804] irq event stamp: 35856 +[ 352.519353] hardirqs last enabled at (35855): [<ffffffff874a1734>] _raw_spin_unlock_irqrestore+0x74/0xc0 +[ 352.529060] hardirqs last disabled at (35856): [<ffffffff87600901>] interrupt_entry+0xb1/0xf0 +[ 352.537731] softirqs last enabled at (162): [<ffffffff87800778>] __do_softirq+0x778/0xaf5 +[ 352.546142] softirqs last disabled at (95): [<ffffffff81467a61>] irq_exit+0x1d1/0x200 +[ 352.554114] CPU: 1 PID: 10431 Comm: syz-executor2 Not tainted 4.16.0+ #4 +[ 352.561560] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 352.570920] RIP: 0010:_raw_spin_unlock_irqrestore+0xa1/0xc0 +[ 352.576631] RSP: 0018:ffff880184db7780 EFLAGS: 00000282 ORIG_RAX: ffffffffffffff13 +[ 352.584344] RAX: dffffc0000000000 RBX: 0000000000000282 RCX: 0000000000000000 +[ 352.591613] RDX: 1ffffffff1162e55 RSI: 0000000000000001 RDI: 0000000000000282 +[ 352.598881] RBP: ffff880184db7790 R08: ffffed0035d21962 R09: 0000000000000000 +[ 352.606150] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801ae90cb08 +[ 352.613418] R13: ffff880184db7810 R14: 0000000000000001 R15: ffff8801cb9a5880 +[ 352.620696] FS: 00007fc943ad8700(0000) GS:ffff8801db100000(0000) knlGS:0000000000000000 +[ 352.628921] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 352.634800] CR2: 00007fc943ad7db8 CR3: 00000001b070d000 CR4: 00000000001406e0 +[ 352.642074] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 352.649346] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 352.656610] Call Trace: +[ 352.659210] snd_virmidi_output_trigger+0x522/0x6c0 +[ 352.674379] snd_rawmidi_kernel_write1+0x519/0x700 +[ 352.679328] snd_rawmidi_write+0x2e2/0xdc0 +[ 352.703684] __vfs_write+0x10b/0x880 +[ 352.751248] vfs_write+0x1f8/0x560 +[ 352.754805] ksys_write+0xf9/0x250 +[ 352.770317] SyS_write+0x24/0x30 +[ 352.777491] do_syscall_64+0x29e/0x9d0 +[ 352.810491] entry_SYSCALL_64_after_hwframe+0x42/0xb7 +[ 352.815683] RIP: 0033:0x455259 +[ 352.818870] RSP: 002b:00007fc943ad7c68 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 +[ 352.826582] RAX: ffffffffffffffda RBX: 00007fc943ad86d4 RCX: 0000000000455259 +[ 352.833855] RDX: 00000000e78e624c RSI: 0000000020000040 RDI: 0000000000000014 +[ 352.841123] RBP: 000000000072c010 R08: 0000000000000000 R09: 0000000000000000 +[ 352.848390] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 352.855661] R13: 00000000000006ca R14: 00000000006fd390 R15: 0000000000000002 +[ 352.882430] Kernel panic - not syncing: softlockup: hung tasks +[ 352.888409] CPU: 1 PID: 10431 Comm: syz-executor2 Tainted: G L 4.16.0+ #4 +[ 352.896551] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 352.905898] Call Trace: +[ 352.908475] <IRQ> +[ 352.910636] dump_stack+0x1b9/0x294 +[ 352.927344] panic+0x22f/0x4de +[ 352.953918] watchdog_timer_fn.cold.5+0xf3/0x126 +[ 352.958685] __hrtimer_run_queues+0x3e3/0x10a0 +[ 353.006124] hrtimer_interrupt+0x286/0x650 +[ 353.010381] smp_apic_timer_interrupt+0x15d/0x710 +[ 353.027757] INFO: rcu_sched detected stalls on CPUs/tasks: +[ 353.029387] 1-....: (1 GPs behind) idle=7a2/1/4611686018427387906 softirq=35919/35920 fqs=20 +[ 353.034918] apic_timer_interrupt+0xf/0x20 +[ 353.034924] </IRQ> +[ 353.034937] RIP: 0010:_raw_spin_unlock_irqrestore+0xa1/0xc0 +[ 353.039231] +[ 353.047962] RSP: 0018:ffff880184db7780 EFLAGS: 00000282 +[ 353.051471] (detected by 0, t=125014 jiffies, g=19098, c=19097, q=977) +[ 353.051482] Sending NMI from CPU 0 to CPUs 1: +[ 353.056298] ORIG_RAX: ffffffffffffff13 +[ 353.056309] RAX: dffffc0000000000 RBX: 0000000000000282 RCX: 0000000000000000 +[ 353.062594] NMI backtrace for cpu 1 +[ 353.062600] CPU: 1 PID: 10431 Comm: syz-executor2 Tainted: G L 4.16.0+ #4 +[ 353.062606] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 353.062611] RIP: 0010:__sanitizer_cov_trace_const_cmp4+0x16/0x20 +[ 353.062616] RSP: 0018:ffff8801db107080 EFLAGS: 00000006 +[ 353.062623] RAX: 0000000000010003 RBX: ffff8801db1071aa RCX: ffffffff87462c86 +[ 353.062629] RDX: 0000000000000003 RSI: 0000000000000009 RDI: 0000000000000005 +[ 353.062634] RBP: ffff8801db107080 R08: ffff8801894c8440 R09: fffffbfff148ee7c +[ 353.062639] R10: fffffbfff148ee7c R11: ffffffff8a4773e0 R12: 0000000000000003 +[ 353.062644] R13: 0000000000003033 R14: 000000000000000a R15: ffffffff8a4773e1 +[ 353.062650] FS: 00007fc943ad8700(0000) GS:ffff8801db100000(0000) knlGS:0000000000000000 +[ 353.062654] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 353.062659] CR2: 00007fc943ad7db8 CR3: 00000001b070d000 CR4: 00000000001406e0 +[ 353.062665] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 353.062670] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 353.062673] Call Trace: +[ 353.062675] <IRQ> +[ 353.062679] put_dec_trunc8+0x266/0x300 +[ 353.062682] put_dec+0x3b/0xf0 +[ 353.062686] number+0xb94/0xc90 +[ 353.062704] vsnprintf+0x13a5/0x1b40 +[ 353.062711] sprintf+0xa7/0xd0 +[ 353.062718] print_prefix+0x26a/0x3f0 +[ 353.062721] msg_print_text+0xca/0x1c0 +[ 353.062725] console_unlock+0x4f5/0x1100 +[ 353.062751] vprintk_emit+0x6ad/0xdd0 +[ 353.062773] vprintk_default+0x28/0x30 +[ 353.062777] vprintk_func+0x7a/0xe7 +[ 353.062780] printk+0x9e/0xba +[ 353.062791] __show_regs.cold.7+0x86/0x54a +[ 353.062799] show_regs_if_on_stack.constprop.7+0x36/0x39 +[ 353.062803] show_trace_log_lvl+0x120/0x25c +[ 353.062811] show_stack+0x38/0x3a +[ 353.062814] dump_stack+0x1b9/0x294 +[ 353.062829] panic+0x22f/0x4de +[ 353.062853] watchdog_timer_fn.cold.5+0xf3/0x126 +[ 353.062857] __hrtimer_run_queues+0x3e3/0x10a0 +[ 353.062898] hrtimer_interrupt+0x286/0x650 +[ 353.062903] smp_apic_timer_interrupt+0x15d/0x710 +[ 353.062930] apic_timer_interrupt+0xf/0x20 +[ 353.062932] </IRQ> +[ 353.062937] RIP: 0010:_raw_spin_unlock_irqrestore+0xa1/0xc0 +[ 353.062941] RSP: 0018:ffff880184db7780 EFLAGS: 00000282 ORIG_RAX: ffffffffffffff13 +[ 353.062950] RAX: dffffc0000000000 RBX: 0000000000000282 RCX: 0000000000000000 +[ 353.062955] RDX: 1ffffffff1162e55 RSI: 0000000000000001 RDI: 0000000000000282 +[ 353.062960] RBP: ffff880184db7790 R08: ffffed0035d21962 R09: 0000000000000000 +[ 353.062965] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801ae90cb08 +[ 353.062971] R13: ffff880184db7810 R14: 0000000000000001 R15: ffff8801cb9a5880 +[ 353.062975] snd_virmidi_output_trigger+0x522/0x6c0 +[ 353.062987] snd_rawmidi_kernel_write1+0x519/0x700 +[ 353.062991] snd_rawmidi_write+0x2e2/0xdc0 +[ 353.063017] __vfs_write+0x10b/0x880 +[ 353.063059] vfs_write+0x1f8/0x560 +[ 353.063062] ksys_write+0xf9/0x250 +[ 353.063076] SyS_write+0x24/0x30 +[ 353.063084] do_syscall_64+0x29e/0x9d0 +[ 353.063112] entry_SYSCALL_64_after_hwframe+0x42/0xb7 +[ 353.063115] RIP: 0033:0x455259 +[ 353.063119] RSP: 002b:00007fc943ad7c68 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 +[ 353.063128] RAX: ffffffffffffffda RBX: 00007fc943ad86d4 RCX: 0000000000455259 +[ 353.063134] RDX: 00000000e78e624c RSI: 0000000020000040 RDI: 0000000000000014 +[ 353.063139] RBP: 000000000072c010 R08: 0000000000000000 R09: 0000000000000000 +[ 353.063144] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 353.063149] R13: 00000000000006ca R14: 00000000006fd390 R15: 0000000000000002 +[ 353.765584] RDX: 1ffffffff1162e55 RSI: 0000000000000001 RDI: 0000000000000282 +[ 353.772854] RBP: ffff880184db7790 R08: ffffed0035d21962 R09: 0000000000000000 +[ 353.780122] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801ae90cb08 +[ 353.787390] R13: ffff880184db7810 R14: 0000000000000001 R15: ffff8801cb9a5880 +[ 353.794699] snd_virmidi_output_trigger+0x522/0x6c0 +[ 353.809878] snd_rawmidi_kernel_write1+0x519/0x700 +[ 353.814825] snd_rawmidi_write+0x2e2/0xdc0 +[ 353.839166] __vfs_write+0x10b/0x880 +[ 353.886693] vfs_write+0x1f8/0x560 +[ 353.890241] ksys_write+0xf9/0x250 +[ 353.905722] SyS_write+0x24/0x30 +[ 353.912894] do_syscall_64+0x29e/0x9d0 +[ 353.945884] entry_SYSCALL_64_after_hwframe+0x42/0xb7 +[ 353.951072] RIP: 0033:0x455259 +[ 353.954261] RSP: 002b:00007fc943ad7c68 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 +[ 353.961976] RAX: ffffffffffffffda RBX: 00007fc943ad86d4 RCX: 0000000000455259 +[ 353.969244] RDX: 00000000e78e624c RSI: 0000000020000040 RDI: 0000000000000014 +[ 353.976513] RBP: 000000000072c010 R08: 0000000000000000 R09: 0000000000000000 +[ 353.983779] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 353.991051] R13: 00000000000006ca R14: 00000000006fd390 R15: 0000000000000002 +[ 354.000710] Dumping ftrace buffer: +[ 354.004343] (ftrace buffer empty) +[ 354.008034] Kernel Offset: disabled +[ 354.011650] Rebooting in 86400 seconds.. diff --git a/pkg/report/testdata/linux/report/266 b/pkg/report/testdata/linux/report/266 new file mode 100644 index 000000000..3dc7fadae --- /dev/null +++ b/pkg/report/testdata/linux/report/266 @@ -0,0 +1,40 @@ +TITLE: BUG: soft lockup in sys_rmdir + +[ 1144.580971] watchdog: BUG: soft lockup - CPU#1 stuck for 134s! [syz-executor0:4569] +[ 1144.589091] Modules linked in: +[ 1144.592335] irq event stamp: 78766212 +[ 1144.596241] hardirqs last enabled at (78766211): [<ffffffff81c7152f>] d_walk+0x80f/0xc80 +[ 1144.604582] hardirqs last disabled at (78766212): [<ffffffff87800905>] interrupt_entry+0xb5/0xf0 +[ 1144.613524] softirqs last enabled at (13779542): [<ffffffff87a00778>] __do_softirq+0x778/0xaf5 +[ 1144.622392] softirqs last disabled at (13779283): [<ffffffff81474901>] irq_exit+0x1d1/0x200 +[ 1144.630891] CPU: 1 PID: 4569 Comm: syz-executor0 Not tainted 4.17.0-rc7+ #38 +[ 1144.639205] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1144.648575] RIP: 0010:check_memory_region+0xc3/0x1b0 +[ 1144.653694] RSP: 0018:ffff8801a14cf930 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 1144.661412] RAX: ffffed0033825dd1 RBX: ffff88019c12ee80 RCX: ffffffff815e0b5e +[ 1144.668708] RDX: 0000000000000000 RSI: 0000000000000004 RDI: ffff88019c12ee80 +[ 1144.676008] RBP: ffff8801a14cf938 R08: ffffed0033825dd1 R09: ffffed0033825dd0 +[ 1144.683284] R10: ffffed0033825dd0 R11: ffff88019c12ee83 R12: 1ffff10034299f29 +[ 1144.690568] R13: dffffc0000000000 R14: ffff8801a14cf9a8 R15: ffff8801a14cfb50 +[ 1144.697845] FS: 0000000000bbe940(0000) GS:ffff8801daf00000(0000) knlGS:0000000000000000 +[ 1144.706087] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 1144.711970] CR2: 0000000000930008 CR3: 00000001a1b76000 CR4: 00000000001406e0 +[ 1144.719244] DR0: 0000000020000000 DR1: 0000000020000000 DR2: 0000000000000000 +[ 1144.726528] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000600 +[ 1144.733967] Call Trace: +[ 1144.740725] do_raw_spin_unlock+0x9e/0x2e0 +[ 1144.758045] _raw_spin_unlock+0x22/0x30 +[ 1144.762022] d_walk+0x451/0xc80 +[ 1144.792735] shrink_dcache_parent+0x179/0x230 +[ 1144.823077] vfs_rmdir+0x202/0x470 +[ 1144.826642] do_rmdir+0x523/0x610 +[ 1144.853253] __x64_sys_rmdir+0x36/0x40 +[ 1144.857239] do_syscall_64+0x1b1/0x800 +[ 1144.886120] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 1144.891437] RIP: 0033:0x455897 +[ 1144.894624] RSP: 002b:00007ffc25471ad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000054 +[ 1144.902336] RAX: ffffffffffffffda RBX: 0000000000000065 RCX: 0000000000455897 +[ 1144.909605] RDX: 0000000000000000 RSI: 00007ffc25473880 RDI: 00007ffc25473880 +[ 1144.916875] RBP: 00007ffc25473880 R08: 0000000000000000 R09: 0000000000000001 +[ 1144.924148] R10: 000000000000000a R11: 0000000000000206 R12: 0000000000bbf940 +[ 1144.931450] R13: 0000000000000000 R14: 00000000000013f1 R15: 00000000000ef6b8 diff --git a/pkg/report/testdata/linux/report/267 b/pkg/report/testdata/linux/report/267 new file mode 100644 index 000000000..05dffdc02 --- /dev/null +++ b/pkg/report/testdata/linux/report/267 @@ -0,0 +1,53 @@ +TITLE: BUG: soft lockup in ipv6_rcv + +[ 1290.104807] watchdog: BUG: soft lockup - CPU#1 stuck for 134s! [syz-executor738:4553] +[ 1290.112830] Modules linked in: +[ 1290.116020] irq event stamp: 13885653 +[ 1290.120062] hardirqs last enabled at (13885652): [<ffffffff878009d5>] restore_regs_and_return_to_kernel+0x0/0x2b +[ 1290.130479] hardirqs last disabled at (13885653): [<ffffffff87800905>] interrupt_entry+0xb5/0xf0 +[ 1290.139538] softirqs last enabled at (13614028): [<ffffffff84df0809>] tun_get_user+0x1dd9/0x4290 +[ 1290.148551] softirqs last disabled at (13614032): [<ffffffff84df1b6f>] tun_get_user+0x313f/0x4290 +[ 1290.158637] CPU: 1 PID: 4553 Comm: syz-executor738 Not tainted 4.17.0-rc3+ #40 +[ 1290.165983] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1290.175400] RIP: 0010:__sanitizer_cov_trace_pc+0x20/0x50 +[ 1290.180861] RSP: 0018:ffff8801d8cfe250 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 1290.188563] RAX: ffff8801d88a8080 RBX: ffff8801d7389e40 RCX: 0000000000000006 +[ 1290.195826] RDX: 0000000000000000 RSI: ffffffff868da4ad RDI: ffff8801c8a53277 +[ 1290.203081] RBP: ffff8801d8cfe250 R08: ffff8801d88a8080 R09: ffff8801d8cfe3e8 +[ 1290.210432] R10: ffffed003b19fc87 R11: ffff8801d8cfe43f R12: ffff8801c8a5327f +[ 1290.217775] R13: 0000000000000000 R14: ffff8801c8a4e5fe R15: ffff8801d8cfe3e8 +[ 1290.225035] FS: 0000000000d88940(0000) GS:ffff8801daf00000(0000) knlGS:0000000000000000 +[ 1290.233337] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 1290.239235] CR2: ffffffffff600400 CR3: 00000001acab3000 CR4: 00000000001406e0 +[ 1290.246677] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 1290.253936] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 1290.261210] Call Trace: +[ 1290.263888] _decode_session6+0xc1d/0x14f0 +[ 1290.268154] __xfrm_decode_session+0x71/0x140 +[ 1290.272668] icmpv6_route_lookup+0x395/0x6e0 +[ 1290.294459] icmp6_send+0x1982/0x2da0 +[ 1290.398009] icmpv6_send+0x17a/0x300 +[ 1290.421143] ip6_input_finish+0x14e1/0x1a30 +[ 1290.444140] ip6_input+0xe1/0x5e0 +[ 1290.456233] ip6_rcv_finish+0x29c/0xa10 +[ 1290.482435] ipv6_rcv+0xeb8/0x2040 +[ 1290.510550] __netif_receive_skb_core+0x2468/0x3650 +[ 1290.597399] __netif_receive_skb+0x2c/0x1e0 +[ 1290.601729] netif_receive_skb_internal+0x126/0x7b0 +[ 1290.629906] napi_gro_frags+0x631/0xc40 +[ 1290.650727] tun_get_user+0x3168/0x4290 +[ 1290.691649] tun_chr_write_iter+0xb9/0x154 +[ 1290.695900] do_iter_readv_writev+0x859/0xa50 +[ 1290.709371] do_iter_write+0x185/0x5f0 +[ 1290.716890] vfs_writev+0x1c7/0x330 +[ 1290.759809] do_writev+0x112/0x2f0 +[ 1290.772027] __x64_sys_writev+0x75/0xb0 +[ 1290.776025] do_syscall_64+0x1b1/0x800 +[ 1290.804960] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 1290.810147] RIP: 0033:0x442c30 +[ 1290.813340] RSP: 002b:00007ffe3fff4978 EFLAGS: 00000246 ORIG_RAX: 0000000000000014 +[ 1290.821081] RAX: ffffffffffffffda RBX: 0000000000000003 RCX: 0000000000442c30 +[ 1290.828385] RDX: 0000000000000001 RSI: 00007ffe3fff49c0 RDI: 00000000000000fc +[ 1290.835646] RBP: 0000000000d89940 R08: 0000000000000000 R09: 00007ffe000087fe +[ 1290.842901] R10: 0000000000000000 R11: 0000000000000246 R12: 585858582e72656c +[ 1290.850156] R13: 6c616b7a79732f2e R14: 0000000000000000 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/268 b/pkg/report/testdata/linux/report/268 new file mode 100644 index 000000000..c6492ea7f --- /dev/null +++ b/pkg/report/testdata/linux/report/268 @@ -0,0 +1,59 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 328.823841] rcu: INFO: rcu_preempt self-detected stall on CPU +[ 328.830097] rcu: 1-...!: (3 GPs behind) idle=19a/1/0x4000000000000002 softirq=12356/12357 fqs=2 +[ 328.839607] rcu: (t=10500 jiffies g=12045 q=155) +[ 328.844567] rcu: rcu_preempt kthread starved for 10492 jiffies! g12045 f0x0 RCU_GP_WAIT_FQS(5) ->state=0x0 ->cpu=0 +[ 328.855046] rcu: RCU grace-period kthread stack dump: +[ 328.860235] rcu_preempt R running task 22872 10 2 0x80000000 +[ 328.867450] Call Trace: +[ 328.870074] __schedule+0x86c/0x1ed0 +[ 328.919601] schedule+0xfe/0x460 +[ 328.950461] schedule_timeout+0x140/0x260 +[ 328.969038] rcu_gp_kthread+0x9d9/0x2310 +[ 329.236894] kthread+0x35a/0x420 +[ 329.248156] ret_from_fork+0x3a/0x50 +[ 329.251894] NMI backtrace for cpu 1 +[ 329.255529] CPU: 1 PID: 8314 Comm: syz-executor2 Not tainted 4.19.0-rc2+ #6 +[ 329.262634] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 329.271989] Call Trace: +[ 329.274578] <IRQ> +[ 329.276738] dump_stack+0x1c4/0x2b4 +[ 329.295554] nmi_cpu_backtrace.cold.3+0x63/0xa2 +[ 329.305441] nmi_trigger_cpumask_backtrace+0x1b3/0x1ed +[ 329.310726] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 329.315961] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 329.325408] print_cpu_stall.cold.78+0x2d3/0x524 +[ 329.374647] rcu_check_callbacks+0xfd9/0x1990 +[ 329.512616] update_process_times+0x2d/0x70 +[ 329.516985] tick_sched_handle+0x9f/0x180 +[ 329.521146] tick_sched_timer+0x45/0x130 +[ 329.525216] __hrtimer_run_queues+0x41c/0x10d0 +[ 329.584253] hrtimer_interrupt+0x313/0x780 +[ 329.588515] smp_apic_timer_interrupt+0x1a1/0x760 +[ 329.655276] apic_timer_interrupt+0xf/0x20 +[ 329.659508] </IRQ> +[ 329.661757] RIP: 0010:kvm_mmu_page_fault+0x2a3/0x1ad0 +[ 329.685879] RSP: 0018:ffff880191beeed0 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 329.693625] RAX: 0000000000000000 RBX: ffff880191b70480 RCX: ffffc900022ce000 +[ 329.700898] RDX: 0000000000000007 RSI: ffffffff8112ec74 RDI: 0000000000000005 +[ 329.708171] RBP: ffff880191bef2b0 R08: ffff8801c44f83c0 R09: ffff880191beed98 +[ 329.715444] R10: ffff880191beeae8 R11: ffff88018d7cf38b R12: 0000000000000001 +[ 329.722721] R13: 0000000000000001 R14: 0000000000000001 R15: 0000000000000f7e +[ 329.842590] handle_ept_violation+0x29e/0x6a0 +[ 329.859146] vmx_handle_exit+0x2f7/0x17f0 +[ 329.884567] vcpu_enter_guest+0x14a9/0x62e0 +[ 329.992063] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 330.001951] kvm_vcpu_ioctl+0x72b/0x1150 +[ 330.061445] do_vfs_ioctl+0x1de/0x1720 +[ 330.102388] ksys_ioctl+0xa9/0xd0 +[ 330.105859] __x64_sys_ioctl+0x73/0xb0 +[ 330.109814] do_syscall_64+0x1b9/0x820 +[ 330.148822] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 330.154015] RIP: 0033:0x457099 +[ 330.176154] RSP: 002b:00007f9e42b44c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 330.183875] RAX: ffffffffffffffda RBX: 00007f9e42b456d4 RCX: 0000000000457099 +[ 330.191150] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000006 +[ 330.198426] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 330.205697] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 330.212969] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/269 b/pkg/report/testdata/linux/report/269 new file mode 100644 index 000000000..18a6692ed --- /dev/null +++ b/pkg/report/testdata/linux/report/269 @@ -0,0 +1,51 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 351.337674] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 351.343969] rcu: (detected by 1, t=10502 jiffies, g=9981, q=283) +[ 351.350228] rcu: All QSes seen, last rcu_preempt kthread activity 10503 (4294972288-4294961785), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 351.363323] syz-executor0 R running task 21280 7720 5511 0x00000000 +[ 351.370540] Call Trace: +[ 351.373124] <IRQ> +[ 351.375304] sched_show_task.cold.83+0x2b6/0x30a +[ 351.411243] print_other_cpu_stall.cold.79+0xa83/0xba5 +[ 351.477062] rcu_check_callbacks+0xafc/0x1990 +[ 351.621883] update_process_times+0x2d/0x70 +[ 351.626218] tick_sched_handle+0x9f/0x180 +[ 351.630382] tick_sched_timer+0x45/0x130 +[ 351.634462] __hrtimer_run_queues+0x41c/0x10d0 +[ 351.693471] hrtimer_interrupt+0x313/0x780 +[ 351.697833] smp_apic_timer_interrupt+0x1a1/0x760 +[ 351.758839] apic_timer_interrupt+0xf/0x20 +[ 351.763076] </IRQ> +[ 351.765322] RIP: 0010:__virt_addr_valid+0xfe/0x230 +[ 351.789242] RSP: 0018:ffff88018a1d68b8 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 351.796963] RAX: dffffc0000000000 RBX: 0000000193be3c00 RCX: ffffc900060ed000 +[ 351.804244] RDX: 1ffffffff1679c04 RSI: ffffffff81386a72 RDI: ffffffff8b3ce020 +[ 351.811523] RBP: ffff88018a1d68d0 R08: ffff8801c071a040 R09: ffffed003277c780 +[ 351.818800] R10: ffffed003277c781 R11: ffff880193be3c0e R12: 0000000000000032 +[ 351.826073] R13: 0000000000000000 R14: 0000000000000000 R15: 000000000000000f +[ 351.842081] __check_object_size+0x15b/0x782 +[ 351.859038] __kvm_read_guest_page+0xda/0x130 +[ 351.863547] kvm_vcpu_read_guest_page+0x40/0x50 +[ 351.868231] kvm_fetch_guest_virt+0x149/0x1c0 +[ 351.876984] __do_insn_fetch_bytes+0x45b/0x950 +[ 351.890976] x86_decode_insn+0x1544/0x54c0 +[ 351.904190] x86_emulate_instruction+0x89a/0x1f90 +[ 351.928622] kvm_mmu_page_fault+0x342/0x1ad0 +[ 352.039266] handle_ept_violation+0x29e/0x6a0 +[ 352.055832] vmx_handle_exit+0x2f7/0x17f0 +[ 352.081257] vcpu_enter_guest+0x14a9/0x62e0 +[ 352.160089] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 352.169984] kvm_vcpu_ioctl+0x72b/0x1150 +[ 352.247217] do_vfs_ioctl+0x1de/0x1720 +[ 352.291956] ksys_ioctl+0xa9/0xd0 +[ 352.295448] __x64_sys_ioctl+0x73/0xb0 +[ 352.299350] do_syscall_64+0x1b9/0x820 +[ 352.343921] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 352.349120] RIP: 0033:0x457099 +[ 352.371250] RSP: 002b:00007f10ce81ac78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 352.378968] RAX: ffffffffffffffda RBX: 00007f10ce81b6d4 RCX: 0000000000457099 +[ 352.386246] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000006 +[ 352.393525] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 352.400800] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 352.408074] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/270 b/pkg/report/testdata/linux/report/270 new file mode 100644 index 000000000..45c7f7ed2 --- /dev/null +++ b/pkg/report/testdata/linux/report/270 @@ -0,0 +1,46 @@ +TITLE: INFO: rcu detected stall in mousedev_write + +[ 489.536147] rcu: INFO: rcu_sched self-detected stall on CPU +[ 489.542259] rcu: 1-....: (104999 ticks this GP) idle=19e/1/0x4000000000000002 softirq=112269/112269 fqs=19579 +[ 489.552836] rcu: (t=105000 jiffies g=225813 q=1973) +[ 489.558034] NMI backtrace for cpu 1 +[ 489.561659] CPU: 1 PID: 21558 Comm: syz-executor5 Not tainted 4.18.0+ #203 +[ 489.568681] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 489.578024] Call Trace: +[ 489.580590] <IRQ> +[ 489.582742] dump_stack+0x1c9/0x2b4 +[ 489.605152] nmi_cpu_backtrace.cold.3+0x48/0x88 +[ 489.615031] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 489.620303] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 489.625482] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 489.634885] print_cpu_stall.cold.78+0x2fb/0x59c +[ 489.660436] rcu_check_callbacks+0xd4a/0x15a0 +[ 489.763624] update_process_times+0x2d/0x70 +[ 489.767934] tick_sched_handle+0x9f/0x180 +[ 489.772076] tick_sched_timer+0x45/0x130 +[ 489.776132] __hrtimer_run_queues+0x3eb/0xff0 +[ 489.825496] hrtimer_interrupt+0x2f3/0x750 +[ 489.829726] smp_apic_timer_interrupt+0x16d/0x6a0 +[ 489.882166] apic_timer_interrupt+0xf/0x20 +[ 489.886385] </IRQ> +[ 489.888622] RIP: 0010:_raw_spin_unlock_irq+0x56/0x70 +[ 489.912610] RSP: 0018:ffff88018eb8faf8 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 +[ 489.920323] RAX: dffffc0000000000 RBX: ffff8801bff42be8 RCX: 0000000000000000 +[ 489.927581] RDX: 1ffffffff0fe379f RSI: 0000000000000001 RDI: ffffffff87f1bcf8 +[ 489.934838] RBP: ffff88018eb8fb00 R08: ffff88018d902000 R09: 0000000000000000 +[ 489.942093] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000c6c5ef +[ 489.949350] R13: dffffc0000000000 R14: 0000000000c6c5ef R15: ffff8801bff42be8 +[ 489.961123] mousedev_write+0x19d/0x840 +[ 489.970770] __vfs_write+0x117/0x9d0 +[ 490.009169] vfs_write+0x1fc/0x560 +[ 490.012705] ksys_write+0x101/0x260 +[ 490.025464] __x64_sys_write+0x73/0xb0 +[ 490.029373] do_syscall_64+0x1b9/0x820 +[ 490.068264] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 490.073439] RIP: 0033:0x457089 +[ 490.095520] RSP: 002b:00007f0d92208c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 +[ 490.103219] RAX: ffffffffffffffda RBX: 00007f0d922096d4 RCX: 0000000000457089 +[ 490.110563] RDX: 00000000ffffffd1 RSI: 00000000200001c0 RDI: 0000000000000005 +[ 490.117820] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 490.125073] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 490.132328] R13: 00000000004d7088 R14: 00000000004ca379 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/271 b/pkg/report/testdata/linux/report/271 new file mode 100644 index 000000000..b202f8740 --- /dev/null +++ b/pkg/report/testdata/linux/report/271 @@ -0,0 +1,57 @@ +TITLE: INFO: rcu detected stall in fuse_dev_release + +[ 629.392020] INFO: rcu_sched detected stalls on CPUs/tasks: +[ 629.397716] (detected by 0, t=105007 jiffies, g=118924, c=118923, q=65) +[ 629.404568] All QSes seen, last rcu_sched kthread activity 105014 (4295296700-4295191686), jiffies_till_next_fqs=3, root ->qsmask 0x0 +[ 629.416688] syz-executor7 R running task 23352 14084 4424 0x00000008 +[ 629.423888] Call Trace: +[ 629.426460] <IRQ> +[ 629.428611] sched_show_task.cold.84+0x27a/0x301 +[ 629.459938] print_other_cpu_stall.cold.77+0x92f/0x9d2 +[ 629.498729] check_cpu_stall.isra.60+0x721/0xf70 +[ 629.570316] rcu_check_callbacks+0x23f/0xcd0 +[ 629.647174] update_process_times+0x2d/0x70 +[ 629.651489] tick_sched_handle+0x9f/0x180 +[ 629.655627] tick_sched_timer+0x45/0x130 +[ 629.659679] __hrtimer_run_queues+0x3eb/0x10c0 +[ 629.708023] hrtimer_interrupt+0x2f3/0x750 +[ 629.712277] smp_apic_timer_interrupt+0x165/0x730 +[ 629.739746] apic_timer_interrupt+0xf/0x20 +[ 629.743963] </IRQ> +[ 629.746192] RIP: 0010:check_memory_region+0x121/0x1b0 +[ 629.770657] RSP: 0018:ffff88018fec7388 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 629.778358] RAX: ffffed0039557a49 RBX: 1ffff10039557a48 RCX: ffffffff815faf29 +[ 629.785618] RDX: 0000000000000000 RSI: 0000000000000004 RDI: ffff8801caabd240 +[ 629.792874] RBP: ffff88018fec73a0 R08: ffffed0039557a49 R09: ffffed0039557a48 +[ 629.800131] R10: ffffed0039557a48 R11: ffff8801caabd243 R12: ffffed0039557a49 +[ 629.807389] R13: 0000000000000001 R14: dffffc0000000000 R15: ffff8801caabd240 +[ 629.820455] kasan_check_read+0x11/0x20 +[ 629.824417] native_queued_spin_lock_slowpath+0x189/0x1200 +[ 629.924285] do_raw_spin_lock+0x1a7/0x200 +[ 629.928436] _raw_spin_lock+0x32/0x40 +[ 629.936113] request_end+0x400/0xb20 +[ 629.948990] end_requests+0x296/0x460 +[ 629.983347] fuse_dev_release+0x18c/0x2e0 +[ 630.004675] __fput+0x355/0x8b0 +[ 630.015712] ____fput+0x15/0x20 +[ 630.018984] task_work_run+0x1ec/0x2a0 +[ 630.031840] exit_to_usermode_loop+0x313/0x370 +[ 630.044964] do_syscall_64+0x6be/0x820 +[ 630.073391] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 630.078571] RIP: 0033:0x456b29 +[ 630.101035] RSP: 002b:00007fae499f2c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000000 +[ 630.108738] RAX: 0000000000000030 RBX: 00007fae499f36d4 RCX: 0000000000456b29 +[ 630.116002] RDX: 0000000000001000 RSI: 0000000020001000 RDI: 0000000000000017 +[ 630.123265] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 630.130534] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 630.137793] R13: 00000000004d2b40 R14: 00000000004c7b8b R15: 0000000000000000 +[ 630.145073] rcu_sched kthread starved for 105755 jiffies! g118924 c118923 f0x2 RCU_GP_WAIT_FQS(3) ->state=0x0 ->cpu=1 +[ 630.155808] RCU grace-period kthread stack dump: +[ 630.160554] rcu_sched R running task 23688 10 2 0x80000000 +[ 630.167746] Call Trace: +[ 630.170332] __schedule+0x87c/0x1ec0 +[ 630.195223] schedule+0xfb/0x450 +[ 630.229166] schedule_timeout+0x140/0x260 +[ 630.242118] rcu_gp_kthread+0x717/0x1c90 +[ 630.389446] kthread+0x345/0x410 +[ 630.401684] ret_from_fork+0x3a/0x50 diff --git a/pkg/report/testdata/linux/report/272 b/pkg/report/testdata/linux/report/272 new file mode 100644 index 000000000..c9c6b7b62 --- /dev/null +++ b/pkg/report/testdata/linux/report/272 @@ -0,0 +1,58 @@ +TITLE: INFO: rcu detected stall in netlink_sendmsg + +[ 480.173025] INFO: rcu_sched self-detected stall on CPU +[ 480.178536] 1-....: (104999 ticks this GP) idle=002/1/4611686018427387906 softirq=125049/125058 fqs=26224 +[ 480.188655] (t=105000 jiffies g=69343 c=69342 q=1302) +[ 480.194010] NMI backtrace for cpu 1 +[ 480.197631] CPU: 1 PID: 29490 Comm: syz-executor5 Not tainted 4.18.0-rc7+ #170 +[ 480.204969] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 480.214301] Call Trace: +[ 480.216863] <IRQ> +[ 480.219013] dump_stack+0x1c9/0x2b4 +[ 480.231770] nmi_cpu_backtrace.cold.4+0x19/0xce +[ 480.241597] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 480.246859] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 480.252036] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 480.261519] check_cpu_stall.isra.60.cold.78+0x36c/0x5a6 +[ 480.326256] rcu_check_callbacks+0x23f/0xcd0 +[ 480.398507] update_process_times+0x2d/0x70 +[ 480.402812] tick_sched_handle+0x9f/0x180 +[ 480.406950] tick_sched_timer+0x45/0x130 +[ 480.410995] __hrtimer_run_queues+0x3eb/0x10c0 +[ 480.459331] hrtimer_interrupt+0x2f3/0x750 +[ 480.463559] smp_apic_timer_interrupt+0x165/0x730 +[ 480.490977] apic_timer_interrupt+0xf/0x20 +[ 480.495190] </IRQ> +[ 480.497429] RIP: 0010:lock_acquire+0x25f/0x540 +[ 480.521174] RSP: 0018:ffff8801999a67c0 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 +[ 480.528871] RAX: dffffc0000000000 RBX: 1ffff10033334cfd RCX: 0000000000000000 +[ 480.536121] RDX: 1ffffffff0fe3615 RSI: 0000000000000000 RDI: 0000000000000286 +[ 480.543371] RBP: ffff8801999a68b0 R08: 0000000000000008 R09: 0000000000000002 +[ 480.550624] R10: ffff880192d5c908 R11: 632564d1cb2a3fed R12: ffff880192d5c080 +[ 480.557874] R13: 0000000000000002 R14: 0000000000000000 R15: 0000000000000000 +[ 480.585903] tipc_sk_lookup+0x131/0xfa0 +[ 480.642175] tipc_nl_publ_dump+0x243/0xfa2 +[ 480.687542] __tipc_nl_compat_dumpit.isra.11+0x20b/0xad0 +[ 480.717699] tipc_nl_compat_sk_dump+0x87c/0xca0 +[ 480.769074] __tipc_nl_compat_dumpit.isra.11+0x337/0xad0 +[ 480.788328] tipc_nl_compat_dumpit+0x1f4/0x440 +[ 480.798089] tipc_nl_compat_recv+0x12b3/0x19a0 +[ 480.825649] genl_family_rcv_msg+0x8a3/0x1140 +[ 480.855652] genl_rcv_msg+0xc6/0x168 +[ 480.859351] netlink_rcv_skb+0x172/0x440 +[ 480.876417] genl_rcv+0x28/0x40 +[ 480.879678] netlink_unicast+0x5a0/0x760 +[ 480.898642] netlink_sendmsg+0xa18/0xfd0 +[ 480.921151] sock_sendmsg+0xd5/0x120 +[ 480.924856] ___sys_sendmsg+0x7fd/0x930 +[ 480.963667] __sys_sendmsg+0x11d/0x290 +[ 480.984618] __x64_sys_sendmsg+0x78/0xb0 +[ 480.988662] do_syscall_64+0x1b9/0x820 +[ 481.012549] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 481.017720] RIP: 0033:0x456a09 +[ 481.040077] RSP: 002b:00007f8665b39c78 EFLAGS: 00000246 ORIG_RAX: 000000000000002e +[ 481.047767] RAX: ffffffffffffffda RBX: 00007f8665b3a6d4 RCX: 0000000000456a09 +[ 481.055026] RDX: 0000000000000000 RSI: 0000000020000000 RDI: 0000000000000022 +[ 481.062281] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 481.076996] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 481.084255] R13: 00000000004d3058 R14: 00000000004c7d3e R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/273 b/pkg/report/testdata/linux/report/273 new file mode 100644 index 000000000..5df0b7ab3 --- /dev/null +++ b/pkg/report/testdata/linux/report/273 @@ -0,0 +1,41 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 631.121467] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 631.127772] rcu: (detected by 0, t=10502 jiffies, g=6833, q=154) +[ 631.134056] rcu: All QSes seen, last rcu_preempt kthread activity 10503 (4295000262-4294989759), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 631.146731] syz-executor4 R running task 22104 7395 5354 0x00000000 +[ 631.153965] Call Trace: +[ 631.156563] <IRQ> +[ 631.158736] sched_show_task.cold.83+0x2b6/0x30a +[ 631.194724] print_other_cpu_stall.cold.79+0xa83/0xba5 +[ 631.261421] rcu_check_callbacks+0xafc/0x1990 +[ 631.401184] update_process_times+0x2d/0x70 +[ 631.405548] tick_sched_handle+0x9f/0x180 +[ 631.409740] tick_sched_timer+0x45/0x130 +[ 631.413842] __hrtimer_run_queues+0x41c/0x10d0 +[ 631.468709] hrtimer_interrupt+0x313/0x780 +[ 631.472986] smp_apic_timer_interrupt+0x1a1/0x760 +[ 631.539804] apic_timer_interrupt+0xf/0x20 +[ 631.544056] </IRQ> +[ 631.546310] RIP: 0010:__srcu_read_lock+0xec/0x160 +[ 631.570098] RSP: 0018:ffff8801bf2bf458 EFLAGS: 00000206 ORIG_RAX: ffffffffffffff13 +[ 631.577828] RAX: 0000000000000001 RBX: 1ffff10037e57e90 RCX: 0000000000000001 +[ 631.585112] RDX: 000060fe24eb2dc0 RSI: 0000000000000000 RDI: ffffc90001fd2c28 +[ 631.592397] RBP: ffff8801bf2bf4e0 R08: ffff8801c0b82080 R09: 0000000000000006 +[ 631.599676] R10: 0000000000000000 R11: ffff8801c0b82080 R12: 1ffff10037e57e8c +[ 631.606953] R13: ffffc90001fd27e0 R14: ffff8801bf2bf4c0 R15: 0000000000000000 +[ 631.622702] vcpu_enter_guest+0x1309/0x62e0 +[ 631.711508] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 631.721422] kvm_vcpu_ioctl+0x72b/0x1150 +[ 631.816167] do_vfs_ioctl+0x1de/0x1720 +[ 631.887384] ksys_ioctl+0xa9/0xd0 +[ 631.890862] __x64_sys_ioctl+0x73/0xb0 +[ 631.894796] do_syscall_64+0x1b9/0x820 +[ 631.933977] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 631.939197] RIP: 0033:0x457099 +[ 631.961342] RSP: 002b:00007ffeca5f9b58 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 631.969086] RAX: ffffffffffffffda RBX: 0000000002395914 RCX: 0000000000457099 +[ 631.976385] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000006 +[ 631.983681] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 631.990979] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 631.998286] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/274 b/pkg/report/testdata/linux/report/274 new file mode 100644 index 000000000..d20c7e56a --- /dev/null +++ b/pkg/report/testdata/linux/report/274 @@ -0,0 +1,60 @@ +TITLE: INFO: rcu detected stall in ipv6_rcv + +[ 842.597024] INFO: rcu_sched self-detected stall on CPU +[ 842.602466] 0-....: (1 GPs behind) idle=222/1/4611686018427387906 softirq=196202/196203 fqs=31202 +[ 842.611869] (t=125000 jiffies g=95129 c=95128 q=2548) +[ 842.617218] NMI backtrace for cpu 0 +[ 842.620820] CPU: 0 PID: 6823 Comm: syz-executor6 Not tainted 4.16.0+ #288 +[ 842.627718] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 842.637043] Call Trace: +[ 842.639601] <IRQ> +[ 842.641730] dump_stack+0x1a7/0x27d +[ 842.662780] nmi_cpu_backtrace+0x1e0/0x220 +[ 842.671464] nmi_trigger_cpumask_backtrace+0x123/0x180 +[ 842.676716] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 842.681881] rcu_dump_cpu_stacks+0x186/0x1d3 +[ 842.686271] check_cpu_stall.isra.63+0xbb8/0x15b0 +[ 842.762673] rcu_check_callbacks+0x21b/0xae0 +[ 842.815807] update_process_times+0x30/0x60 +[ 842.820105] tick_sched_handle+0x85/0x160 +[ 842.824229] tick_sched_timer+0x42/0x120 +[ 842.828264] __hrtimer_run_queues+0x39c/0xec0 +[ 842.874887] hrtimer_interrupt+0x2a5/0x6f0 +[ 842.879110] smp_apic_timer_interrupt+0x14a/0x700 +[ 842.914536] apic_timer_interrupt+0xf/0x20 +[ 842.918740] </IRQ> +[ 842.920956] RIP: 0010:__sanitizer_cov_trace_pc+0x0/0x50 +[ 842.926292] RSP: 0018:ffff8801aabde340 EFLAGS: 00000287 ORIG_RAX: ffffffffffffff12 +[ 842.933978] RAX: 0000000000000007 RBX: ffff8801c81331c0 RCX: 000000000000fde1 +[ 842.941221] RDX: 0000000000000000 RSI: 0000000000004699 RDI: ffff8801c8133244 +[ 842.948465] RBP: ffff8801aabde3d0 R08: 0000000000000000 R09: ffff8801aabde6a8 +[ 842.955718] R10: 000000000000000b R11: ffffed003557bcdf R12: 000000000000fde1 +[ 842.962962] R13: ffff8801aabde6a8 R14: 00000000000046c8 R15: dffffc0000000000 +[ 842.979432] __xfrm_decode_session+0x68/0x110 +[ 842.983904] __xfrm_policy_check+0x19e/0x2300 +[ 843.040856] ip6_input_finish+0x1288/0x17b0 +[ 843.068947] ip6_input+0xdb/0x560 +[ 843.089410] ip6_mc_input+0x3a8/0xb10 +[ 843.096979] ip6_rcv_finish+0x297/0x8c0 +[ 843.121561] ipv6_rcv+0xf45/0x1fc0 +[ 843.145285] __netif_receive_skb_core+0x1a52/0x3480 +[ 843.236444] __netif_receive_skb+0x2c/0x1b0 +[ 843.245211] netif_receive_skb_internal+0x10b/0x670 +[ 843.266287] netif_receive_skb+0xae/0x390 +[ 843.288664] tun_rx_batched.isra.50+0x5ee/0x870 +[ 843.306989] tun_get_user+0x299b/0x3b50 +[ 843.363324] tun_chr_write_iter+0xb9/0x160 +[ 843.367536] do_iter_readv_writev+0x55c/0x830 +[ 843.380795] do_iter_write+0x154/0x540 +[ 843.392735] compat_writev+0x225/0x420 +[ 843.420428] do_compat_writev+0x115/0x220 +[ 843.440528] compat_SyS_writev+0x26/0x30 +[ 843.448867] do_fast_syscall_32+0x3ec/0xf9f +[ 843.486226] entry_SYSENTER_compat+0x70/0x7f +[ 843.490607] RIP: 0023:0xf7ff2c99 +[ 843.493944] RSP: 002b:00000000f5fee044 EFLAGS: 00000292 ORIG_RAX: 0000000000000092 +[ 843.501630] RAX: ffffffffffffffda RBX: 00000000000000fc RCX: 00000000f5fee094 +[ 843.508882] RDX: 0000000000000001 RSI: 000000000813af00 RDI: 00000000ffffffff +[ 843.516133] RBP: 00000000f5fee158 R08: 0000000000000000 R09: 0000000000000000 +[ 843.523377] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000 +[ 843.530621] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/275 b/pkg/report/testdata/linux/report/275 new file mode 100644 index 000000000..bdeb527fd --- /dev/null +++ b/pkg/report/testdata/linux/report/275 @@ -0,0 +1,45 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_release + +[ 401.753130] rcu: INFO: rcu_sched detected stalls on CPUs/tasks: +[ 401.759317] rcu: (detected by 0, t=105002 jiffies, g=89793, q=195) +[ 401.765737] rcu: All QSes seen, last rcu_sched kthread activity 105002 (4295067669-4294962667), jiffies_till_next_fqs=3, root ->qsmask 0x0 +[ 401.778318] syz-executor5 R running task 24488 9936 4748 0x00000008 +[ 401.785560] Call Trace: +[ 401.788141] <IRQ> +[ 401.790301] sched_show_task.cold.85+0x273/0x2d5 +[ 401.821799] print_other_cpu_stall.cold.79+0x8e0/0x91d +[ 401.860568] rcu_check_callbacks+0x986/0x15a0 +[ 401.964031] update_process_times+0x2d/0x70 +[ 401.968355] tick_sched_handle+0x9f/0x180 +[ 401.972517] tick_sched_timer+0x45/0x130 +[ 401.976581] __hrtimer_run_queues+0x3eb/0xff0 +[ 402.026193] hrtimer_interrupt+0x2f3/0x750 +[ 402.030445] smp_apic_timer_interrupt+0x16d/0x6a0 +[ 402.083022] apic_timer_interrupt+0xf/0x20 +[ 402.087252] </IRQ> +[ 402.089510] RIP: 0010:smp_call_function_single+0x3f5/0x5c0 +[ 402.114059] RSP: 0018:ffff8801cfb8f3a0 EFLAGS: 00000293 ORIG_RAX: ffffffffffffff13 +[ 402.121773] RAX: ffff8801cfb8f418 RBX: ffff8801cfb8f418 RCX: ffffffff8170865c +[ 402.129044] RDX: 0000000000000000 RSI: ffffffff8170860b RDI: 0000000000000005 +[ 402.136314] RBP: ffff8801cfb8f4f0 R08: ffff8801bcb92140 R09: ffffed003b625b98 +[ 402.143582] R10: ffffed003b625b98 R11: ffff8801db12dcc7 R12: ffffed0039f71e90 +[ 402.150856] R13: dffffc0000000000 R14: 0000000000000000 R15: 0000000000000001 +[ 402.191483] vmx_vcpu_load+0x873/0xfe0 +[ 402.235986] kvm_arch_vcpu_load+0x22b/0x940 +[ 402.244821] vcpu_load+0x32/0x40 +[ 402.248189] kvm_arch_destroy_vm+0x1f9/0x7c0 +[ 402.284984] kvm_put_kvm+0x73f/0x1060 +[ 402.317058] kvm_vcpu_release+0x7b/0xa0 +[ 402.321039] __fput+0x38a/0xa40 +[ 402.354473] ____fput+0x15/0x20 +[ 402.357755] task_work_run+0x1e8/0x2a0 +[ 402.374215] exit_to_usermode_loop+0x318/0x380 +[ 402.389202] do_syscall_64+0x6be/0x820 +[ 402.423222] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 402.428415] RIP: 0033:0x410c41 +[ 402.450537] RSP: 002b:00007ffc6fee9a70 EFLAGS: 00000293 ORIG_RAX: 0000000000000003 +[ 402.458282] RAX: 0000000000000000 RBX: 000000000000000b RCX: 0000000000410c41 +[ 402.465556] RDX: 0000000000000000 RSI: 00000000007305f0 RDI: 000000000000000a +[ 402.472826] RBP: 0000000000000000 R08: ffffffffffffffff R09: ffffffffffffffff +[ 402.480102] R10: 00007ffc6fee99a0 R11: 0000000000000293 R12: 0000000000000009 +[ 402.487376] R13: 00000000000485c6 R14: 0000000000000052 R15: badc0ffeebadface diff --git a/pkg/report/testdata/linux/report/276 b/pkg/report/testdata/linux/report/276 new file mode 100644 index 000000000..037de1360 --- /dev/null +++ b/pkg/report/testdata/linux/report/276 @@ -0,0 +1,46 @@ +TITLE: INFO: rcu detected stall in __cleanup_mnt + +[ 396.319024] INFO: rcu_sched detected stalls on CPUs/tasks: +[ 396.324708] (detected by 0, t=105007 jiffies, g=54385, c=54384, q=120) +[ 396.331479] All QSes seen, last rcu_sched kthread activity 105014 (4295063628-4294958614), jiffies_till_next_fqs=3, root ->qsmask 0x0 +[ 396.343605] syz-executor5 R running task 25224 22239 4482 0x00000008 +[ 396.350812] Call Trace: +[ 396.353393] <IRQ> +[ 396.355560] sched_show_task.cold.84+0x27a/0x301 +[ 396.386938] print_other_cpu_stall.cold.77+0x92f/0x9d2 +[ 396.425896] check_cpu_stall.isra.60+0x721/0xf70 +[ 396.497958] rcu_check_callbacks+0x23f/0xcd0 +[ 396.575053] update_process_times+0x2d/0x70 +[ 396.579598] tick_sched_handle+0x9f/0x180 +[ 396.583746] tick_sched_timer+0x45/0x130 +[ 396.587826] __hrtimer_run_queues+0x3eb/0x10c0 +[ 396.636316] hrtimer_interrupt+0x2f3/0x750 +[ 396.640571] smp_apic_timer_interrupt+0x165/0x730 +[ 396.668106] apic_timer_interrupt+0xf/0x20 +[ 396.672346] </IRQ> +[ 396.674589] RIP: 0010:fuse_abort_conn+0xb1f/0x1580 +[ 396.698893] RSP: 0018:ffff880198357618 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 396.706602] RAX: 0000000000040000 RBX: dffffc0000000000 RCX: ffffc9000d4c7000 +[ 396.713886] RDX: 0000000000040000 RSI: ffffffff8267f418 RDI: ffff8801d7f27270 +[ 396.721154] RBP: ffff8801983579e0 R08: ffffed003afe4e50 R09: ffffed003afe4e4f +[ 396.728422] R10: ffffed003afe4e4f R11: ffff8801d7f2727b R12: ffff8801ae944108 +[ 396.735689] R13: ffff880198357738 R14: ffff8801983579b8 R15: dffffc0000000000 +[ 396.815744] fuse_put_super+0x123/0x440 +[ 396.841988] generic_shutdown_super+0x1bf/0x530 +[ 396.861913] kill_anon_super+0x3c/0x50 +[ 396.865801] fuse_kill_sb_anon+0x90/0xb0 +[ 396.869866] deactivate_locked_super+0x97/0x100 +[ 396.874546] deactivate_super+0x193/0x1c0 +[ 396.882779] cleanup_mnt+0xbf/0x160 +[ 396.886406] __cleanup_mnt+0x16/0x20 +[ 396.890123] task_work_run+0x1ec/0x2a0 +[ 396.907192] exit_to_usermode_loop+0x313/0x370 +[ 396.920274] do_syscall_64+0x6be/0x820 +[ 396.948792] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 396.953981] RIP: 0033:0x456cb9 +[ 396.976544] RSP: 002b:00007fc8f9ed9c78 EFLAGS: 00000246 ORIG_RAX: 00000000000000a6 +[ 396.984286] RAX: 0000000000000000 RBX: 00007fc8f9eda6d4 RCX: 0000000000456cb9 +[ 396.991568] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000020000100 +[ 396.998834] RBP: 0000000000930280 R08: 0000000000000000 R09: 0000000000000000 +[ 397.006100] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 397.013366] R13: 00000000004d6360 R14: 00000000004c9ab8 R15: 0000000000000003 diff --git a/pkg/report/testdata/linux/report/277 b/pkg/report/testdata/linux/report/277 new file mode 100644 index 000000000..fa3d208d0 --- /dev/null +++ b/pkg/report/testdata/linux/report/277 @@ -0,0 +1,54 @@ +TITLE: INFO: rcu detected stall in ext4_file_write_iter + +[ 587.513972] rcu: INFO: rcu_sched self-detected stall on CPU +[ 587.520042] rcu: 0-....: (105000 ticks this GP) idle=76a/1/0x4000000000000002 softirq=144952/144952 fqs=26032 +[ 587.530723] rcu: (t=105010 jiffies g=334085 q=1885) +[ 587.535924] NMI backtrace for cpu 0 +[ 587.539556] CPU: 0 PID: 30128 Comm: syz-executor6 Not tainted 4.18.0+ #197 +[ 587.546568] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 587.555920] Call Trace: +[ 587.558509] <IRQ> +[ 587.560674] dump_stack+0x1c9/0x2b4 +[ 587.578675] nmi_cpu_backtrace.cold.3+0x48/0x88 +[ 587.588548] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 587.593830] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 587.599018] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 587.608455] print_cpu_stall.cold.78+0x2fb/0x59c +[ 587.638910] rcu_check_callbacks+0xd93/0x1660 +[ 587.732161] update_process_times+0x2d/0x70 +[ 587.736486] tick_sched_handle+0x9f/0x180 +[ 587.740649] tick_sched_timer+0x45/0x130 +[ 587.744711] __hrtimer_run_queues+0x3eb/0x10c0 +[ 587.785214] hrtimer_interrupt+0x2f3/0x750 +[ 587.789483] smp_apic_timer_interrupt+0x16d/0x730 +[ 587.817071] apic_timer_interrupt+0xf/0x20 +[ 587.821308] </IRQ> +[ 587.823563] RIP: 0010:lock_acquire+0x25f/0x540 +[ 587.847050] RSP: 0018:ffff8801912ee5e8 EFLAGS: 00000282 ORIG_RAX: ffffffffffffff13 +[ 587.854764] RAX: dffffc0000000000 RBX: 1ffff1003225dcc2 RCX: 0000000000000000 +[ 587.862032] RDX: 1ffffffff0fe374d RSI: 0000000000000000 RDI: 0000000000000282 +[ 587.869301] RBP: ffff8801912ee6d8 R08: 0000000000000008 R09: 0000000000000004 +[ 587.876569] R10: ffff8801bb228d58 R11: 5f54006f34535a0e R12: ffff8801bb228480 +[ 587.883836] R13: 0000000000000002 R14: 0000000000000000 R15: 0000000000000000 +[ 587.903344] balance_dirty_pages_ratelimited+0x187f/0x2200 +[ 587.991035] generic_perform_write+0x551/0x6c0 +[ 588.027011] __generic_file_write_iter+0x26e/0x630 +[ 588.031952] ext4_file_write_iter+0x390/0x1450 +[ 588.082118] __vfs_write+0x6af/0x9d0 +[ 588.102237] __kernel_write+0x10c/0x380 +[ 588.106225] do_acct_process+0x1148/0x1660 +[ 588.118181] acct_process+0x6b9/0x87d +[ 588.151604] do_exit+0x1afd/0x2760 +[ 588.255303] do_group_exit+0x177/0x440 +[ 588.277377] get_signal+0x88e/0x1970 +[ 588.310755] do_signal+0x9c/0x21c0 +[ 588.350143] exit_to_usermode_loop+0x2e5/0x380 +[ 588.365138] do_syscall_64+0x6be/0x820 +[ 588.397707] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 588.402900] RIP: 0033:0x457089 +[ 588.425533] RSP: 002b:00007fe66d9d4c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000003 +[ 588.433254] RAX: 0000000000000000 RBX: 00007fe66d9d56d4 RCX: 0000000000457089 +[ 588.440538] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000004 +[ 588.447815] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 588.455086] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 588.462371] R13: 00000000004cb460 R14: 00000000004ee072 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/278 b/pkg/report/testdata/linux/report/278 new file mode 100644 index 000000000..9c27db5cf --- /dev/null +++ b/pkg/report/testdata/linux/report/278 @@ -0,0 +1,59 @@ +TITLE: INFO: rcu detected stall in sys_futex + +[ 246.173515] rcu: INFO: rcu_sched self-detected stall on CPU +[ 246.179319] rcu: 1-...!: (1 ticks this GP) idle=2e6/1/0x4000000000000002 softirq=21091/21091 fqs=0 +[ 246.188580] rcu: (t=110325 jiffies g=45621 q=219) +[ 246.193603] rcu: rcu_sched kthread starved for 110325 jiffies! g45621 f0x0 RCU_GP_WAIT_FQS(5) ->state=0x402 ->cpu=1 +[ 246.204159] rcu: RCU grace-period kthread stack dump: +[ 246.209334] rcu_sched I23480 10 2 0x80000000 +[ 246.214997] Call Trace: +[ 246.217708] __schedule+0x87c/0x1ec0 +[ 246.247401] schedule+0xfb/0x450 +[ 246.277246] schedule_timeout+0x140/0x260 +[ 246.294402] rcu_gp_kthread+0x743/0x1d20 +[ 246.481639] kthread+0x35a/0x420 +[ 246.493143] ret_from_fork+0x3a/0x50 +[ 246.496878] NMI backtrace for cpu 1 +[ 246.500524] CPU: 1 PID: 4356 Comm: syz-fuzzer Not tainted 4.18.0-rc7-next-20180803+ #31 +[ 246.508667] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 246.518010] Call Trace: +[ 246.520578] <IRQ> +[ 246.522762] dump_stack+0x1c9/0x2b4 +[ 246.540779] nmi_cpu_backtrace.cold.3+0x48/0x88 +[ 246.550627] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 246.555897] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 246.561086] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 246.570531] print_cpu_stall.cold.78+0x2fb/0x59c +[ 246.596748] rcu_check_callbacks+0xd93/0x1660 +[ 246.719004] update_process_times+0x2d/0x70 +[ 246.723352] tick_sched_handle+0x9f/0x180 +[ 246.727495] tick_sched_timer+0x45/0x130 +[ 246.731565] __hrtimer_run_queues+0x3eb/0x10c0 +[ 246.776185] hrtimer_interrupt+0x2f3/0x750 +[ 246.780455] smp_apic_timer_interrupt+0x165/0x730 +[ 246.807990] apic_timer_interrupt+0xf/0x20 +[ 246.812208] </IRQ> +[ 246.814441] RIP: 0010:_raw_spin_unlock_irq+0x56/0x70 +[ 246.838429] RSP: 0018:ffff8801ad6ef090 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 +[ 246.846148] RAX: dffffc0000000000 RBX: ffff8801db12ca40 RCX: ffffffff81606487 +[ 246.853406] RDX: 1ffffffff0fe361f RSI: 0000000000000004 RDI: ffffffff87f1b0f8 +[ 246.860677] RBP: ffff8801ad6ef098 R08: ffffed003b625949 R09: ffffed003b625948 +[ 246.867934] R10: ffffed003b625948 R11: ffff8801db12ca43 R12: ffff8801ad502180 +[ 246.875212] R13: ffff8801d9f18380 R14: ffff88019c2e98c0 R15: dffffc0000000000 +[ 246.886921] finish_task_switch+0x1d3/0x870 +[ 246.965692] __schedule+0x884/0x1ec0 +[ 247.014854] schedule+0xfb/0x450 +[ 247.049482] futex_wait_queue_me+0x3f9/0x840 +[ 247.100651] futex_wait+0x45b/0xa20 +[ 247.125352] do_futex+0x336/0x27d0 +[ 247.229783] __x64_sys_futex+0x472/0x6a0 +[ 247.247689] do_syscall_64+0x1b9/0x820 +[ 247.280793] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 247.285980] RIP: 0033:0x45ddf3 +[ 247.308057] RSP: 002b:000000c420035e80 EFLAGS: 00000246 ORIG_RAX: 00000000000000ca +[ 247.315756] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 000000000045ddf3 +2018/08/03 14:36:29 Manager.Poll call failed: connection is shut down +[ 247.323013] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000001448298 +[ 247.330271] RBP: 000000c420035ec8 R08: 0000000000000000 R09: 0000000000000000 +[ 247.337529] R10: 000000c420035eb8 R11: 0000000000000246 R12: 0000000000430130 +[ 247.344791] R13: 00000000000000f1 R14: 0000000000000011 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/279 b/pkg/report/testdata/linux/report/279 new file mode 100644 index 000000000..50d232b8e --- /dev/null +++ b/pkg/report/testdata/linux/report/279 @@ -0,0 +1,47 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 1057.417927] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 1057.424666] rcu: (detected by 1, t=10502 jiffies, g=7113, q=509) +[ 1057.431125] rcu: All QSes seen, last rcu_preempt kthread activity 10498 (4295042895-4295032397), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 1057.444299] syz-executor0 R running task 22808 7363 5346 0x00000000 +[ 1057.451771] Call Trace: +[ 1057.454712] <IRQ> +[ 1057.457168] sched_show_task.cold.83+0x2b6/0x30a +[ 1057.495301] print_other_cpu_stall.cold.79+0xa83/0xba5 +[ 1057.554036] rcu_check_callbacks+0xafc/0x1990 +[ 1057.687133] update_process_times+0x2d/0x70 +[ 1057.691573] tick_sched_handle+0x9f/0x180 +[ 1057.695766] tick_sched_timer+0x45/0x130 +[ 1057.699875] __hrtimer_run_queues+0x41c/0x10d0 +[ 1057.749858] hrtimer_interrupt+0x313/0x780 +[ 1057.754284] smp_apic_timer_interrupt+0x1a1/0x760 +[ 1057.821414] apic_timer_interrupt+0xf/0x20 +[ 1057.825660] </IRQ> +[ 1057.827917] RIP: 0010:lock_acquire+0x268/0x520 +[ 1057.851437] RSP: 0018:ffff8801c664eb20 EFLAGS: 00000282 ORIG_RAX: ffffffffffffff13 +[ 1057.859184] RAX: dffffc0000000000 RBX: 1ffff10038cc9d69 RCX: 0000000000000000 +[ 1057.866490] RDX: 1ffffffff12a4535 RSI: 0000000000000000 RDI: 0000000000000282 +[ 1057.873788] RBP: ffff8801c664ec10 R08: ffff8801c10e6e70 R09: 0000000000000002 +[ 1057.881090] R10: ffff8801c10e6e50 R11: f333abdc872bc457 R12: ffff8801c10e6580 +[ 1057.888382] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000000 +[ 1057.916107] _raw_spin_lock+0x2d/0x40 +[ 1057.925256] kvm_mmu_unprotect_page+0xf9/0x450 +[ 1057.938957] x86_emulate_instruction+0x635/0x1fc0 +[ 1057.970189] kvm_mmu_page_fault+0x396/0x1b30 +[ 1058.076393] handle_ept_violation+0x29e/0x6a0 +[ 1058.092996] vmx_handle_exit+0x2f7/0x17e0 +[ 1058.133773] vcpu_enter_guest+0x14a9/0x62e0 +[ 1058.232876] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 1058.242845] kvm_vcpu_ioctl+0x72b/0x1150 +[ 1058.347536] do_vfs_ioctl+0x1de/0x1720 +[ 1058.404376] ksys_ioctl+0xa9/0xd0 +[ 1058.407861] __x64_sys_ioctl+0x73/0xb0 +[ 1058.411768] do_syscall_64+0x1b9/0x820 +[ 1058.450911] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 1058.456119] RIP: 0033:0x457099 +[ 1058.478267] RSP: 002b:00007fff8bfb2748 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 1058.486005] RAX: ffffffffffffffda RBX: 000000000212b914 RCX: 0000000000457099 +[ 1058.493299] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000006 +[ 1058.500585] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 1058.507875] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 1058.515159] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/280 b/pkg/report/testdata/linux/report/280 new file mode 100644 index 000000000..6900184cd --- /dev/null +++ b/pkg/report/testdata/linux/report/280 @@ -0,0 +1,49 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 885.637820] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 885.644147] rcu: (detected by 1, t=10502 jiffies, g=132945, q=115) +[ 885.650574] rcu: All QSes seen, last rcu_preempt kthread activity 10498 (4295025713-4295015215), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 885.663229] syz-executor7 R running task 21280 8024 5432 0x00000002 +[ 885.670447] Call Trace: +[ 885.673034] <IRQ> +[ 885.675201] sched_show_task.cold.83+0x2b6/0x30a +[ 885.711132] print_other_cpu_stall.cold.79+0xa83/0xba5 +[ 885.764802] rcu_check_callbacks+0xafc/0x1990 +[ 885.897413] update_process_times+0x2d/0x70 +[ 885.901759] tick_sched_handle+0x9f/0x180 +[ 885.905940] tick_sched_timer+0x45/0x130 +[ 885.910010] __hrtimer_run_queues+0x41c/0x10d0 +[ 885.966004] hrtimer_interrupt+0x313/0x780 +[ 885.970274] smp_apic_timer_interrupt+0x1a1/0x760 +[ 886.031595] apic_timer_interrupt+0xf/0x20 +[ 886.035836] </IRQ> +[ 886.038088] RIP: 0010:vmx_read_guest_seg_ar+0x127/0x270 +[ 886.062366] RSP: 0018:ffff8801a035ea40 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 +[ 886.070085] RAX: 0000000000000093 RBX: 0000000000000002 RCX: ffffc90004d15000 +[ 886.077363] RDX: 0000000000004818 RSI: ffffffff811c3301 RDI: ffffffff8803e9ac +[ 886.084637] RBP: ffff8801a035ea68 R08: ffff8801bfe0a140 R09: ffff8801c1f7c480 +[ 886.091910] R10: ffffed00383ef891 R11: ffff8801c1f7c48b R12: ffff8801c1f78a80 +[ 886.099184] R13: 0000000000004818 R14: 0000000000000000 R15: 0000000000100000 +[ 886.111255] vmx_get_cpl+0x5b/0x90 +[ 886.114813] kvm_fetch_guest_virt+0x65/0x1c0 +[ 886.123486] __do_insn_fetch_bytes+0x45b/0x950 +[ 886.136686] x86_decode_insn+0x1544/0x54c0 +[ 886.149892] x86_emulate_instruction+0x833/0x1fc0 +[ 886.171431] kvm_mmu_page_fault+0x396/0x1b30 +[ 886.299270] handle_ept_violation+0x29e/0x6a0 +[ 886.320472] vmx_handle_exit+0x2f7/0x17e0 +[ 886.345943] vcpu_enter_guest+0x14a9/0x62e0 +[ 886.447583] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 886.457476] kvm_vcpu_ioctl+0x72b/0x1150 +[ 886.510921] do_vfs_ioctl+0x1de/0x1720 +[ 886.567519] ksys_ioctl+0xa9/0xd0 +[ 886.570985] __x64_sys_ioctl+0x73/0xb0 +[ 886.574886] do_syscall_64+0x1b9/0x820 +[ 886.619519] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 886.624716] RIP: 0033:0x457099 +[ 886.646837] RSP: 002b:00007f84c55d1c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 886.654558] RAX: ffffffffffffffda RBX: 00007f84c55d26d4 RCX: 0000000000457099 +[ 886.661834] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000006 +[ 886.669108] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 886.676383] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 886.683661] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/281 b/pkg/report/testdata/linux/report/281 new file mode 100644 index 000000000..6a6616ef7 --- /dev/null +++ b/pkg/report/testdata/linux/report/281 @@ -0,0 +1,55 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 557.780627] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 557.786921] rcu: (detected by 0, t=10502 jiffies, g=52381, q=77) +[ 557.793179] rcu: All QSes seen, last rcu_preempt kthread activity 10503 (4294992931-4294982428), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 557.805832] syz-executor1 R running task 21584 17898 5513 0x00000000 +[ 557.813042] Call Trace: +[ 557.815629] <IRQ> +[ 557.817796] sched_show_task.cold.83+0x2b6/0x30a +[ 557.853686] print_other_cpu_stall.cold.79+0xa83/0xba5 +[ 557.919480] rcu_check_callbacks+0xafc/0x1990 +[ 558.059280] update_process_times+0x2d/0x70 +[ 558.063624] tick_sched_handle+0x9f/0x180 +[ 558.067785] tick_sched_timer+0x45/0x130 +[ 558.071860] __hrtimer_run_queues+0x41c/0x10d0 +[ 558.126464] hrtimer_interrupt+0x313/0x780 +[ 558.130798] smp_apic_timer_interrupt+0x1a1/0x760 +[ 558.191678] apic_timer_interrupt+0xf/0x20 +[ 558.195920] </IRQ> +[ 558.198167] RIP: 0010:do_error_trap+0x270/0x4d0 +[ 558.221781] RSP: 0018:ffff88018c7f68e8 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 +[ 558.229505] RAX: dffffc0000000000 RBX: ffff88018c7f6a38 RCX: 1ffff10031358d4c +[ 558.236780] RDX: 1ffffffff12a4537 RSI: ffff880189ac6a68 RDI: ffffffff895229b8 +[ 558.244058] RBP: ffff88018c7f6a18 R08: ffff880189ac6a60 R09: 0000000000000006 +[ 558.251343] R10: 0000000000000000 R11: ffff880189ac61c0 R12: 0000000000000000 +[ 558.258622] R13: ffff88018c7f69f0 R14: 0000000000000008 R15: 1ffff100318fed22 +[ 558.311744] do_divide_error+0x18/0x20 +[ 558.315648] divide_error+0x14/0x20 +[ 558.319295] RIP: 0010:em_idiv_ex+0x0/0x8 +[ 558.342278] RSP: 0018:ffff88018c7f6ae0 EFLAGS: 00010202 +[ 558.347666] RAX: 0000000000000000 RBX: ffff88018ae6c960 RCX: 0000000000000000 +[ 558.354939] RDX: 0000000000000000 RSI: ffffffff811474e8 RDI: 0000000000000200 +[ 558.362215] RBP: ffff88018c7f6b18 R08: ffff880189ac61c0 R09: 1ffffffff12b43cd +[ 558.369489] R10: ffffed003b5c4732 R11: ffff8801dae23993 R12: 0000000000000000 +[ 558.376760] R13: ffffffff811474e8 R14: 0000000000000001 R15: ffffffff811474e8 +[ 558.401977] x86_emulate_insn+0x1efd/0x5020 +[ 558.437698] x86_emulate_instruction+0x665/0x1f90 +[ 558.460608] kvm_mmu_page_fault+0x342/0x1ad0 +[ 558.570767] handle_ept_violation+0x29e/0x6a0 +[ 558.587169] vmx_handle_exit+0x2f7/0x17f0 +[ 558.603946] vcpu_enter_guest+0x14a9/0x62e0 +[ 558.687489] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 558.697387] kvm_vcpu_ioctl+0x72b/0x1150 +[ 558.762259] do_vfs_ioctl+0x1de/0x1720 +[ 558.790202] ksys_ioctl+0xa9/0xd0 +[ 558.793668] __x64_sys_ioctl+0x73/0xb0 +[ 558.797566] do_syscall_64+0x1b9/0x820 +[ 558.842137] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 558.847341] RIP: 0033:0x457099 +[ 558.869448] RSP: 002b:00007fe96d7efc78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 558.877169] RAX: ffffffffffffffda RBX: 00007fe96d7f06d4 RCX: 0000000000457099 +[ 558.884442] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000009 +[ 558.891714] RBP: 0000000000930140 R08: 0000000000000000 R09: 0000000000000000 +[ 558.898989] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 558.906264] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000001 diff --git a/pkg/report/testdata/linux/report/282 b/pkg/report/testdata/linux/report/282 new file mode 100644 index 000000000..35fefb6ba --- /dev/null +++ b/pkg/report/testdata/linux/report/282 @@ -0,0 +1,43 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 618.102956] overlayfs: failed to resolve './file1': -2 +[ 724.054784] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 724.061136] rcu: (detected by 1, t=10502 jiffies, g=71473, q=34) +[ 724.067416] rcu: All QSes seen, last rcu_preempt kthread activity 10503 (4295009460-4294998957), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 724.080087] syz-executor1 R running task 22672 24999 5368 0x00000000 +[ 724.087341] Call Trace: +[ 724.089931] <IRQ> +[ 724.092107] sched_show_task.cold.83+0x2b6/0x30a +[ 724.128077] print_other_cpu_stall.cold.79+0xa83/0xba5 +[ 724.199489] rcu_check_callbacks+0xafc/0x1990 +[ 724.345408] update_process_times+0x2d/0x70 +[ 724.349750] tick_sched_handle+0x9f/0x180 +[ 724.353919] tick_sched_timer+0x45/0x130 +[ 724.357996] __hrtimer_run_queues+0x41c/0x10d0 +[ 724.412730] hrtimer_interrupt+0x313/0x780 +[ 724.417102] smp_apic_timer_interrupt+0x1a1/0x760 +[ 724.477953] apic_timer_interrupt+0xf/0x20 +[ 724.482196] </IRQ> +[ 724.484449] RIP: 0010:debug_lockdep_rcu_enabled.part.3+0x0/0x60 +[ 724.509432] RSP: 0018:ffff88017575f3a0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 +[ 724.517157] RAX: 0000000000000003 RBX: ffffffff89936580 RCX: 0000000000000002 +[ 724.524434] RDX: 0000000000000001 RSI: ffffffff8120f03f RDI: ffffffff8a314a50 +[ 724.531735] RBP: ffff88017575f3b0 R08: ffff8801bbe52580 R09: 0000000000000001 +[ 724.539010] R10: ffff8801bbe52e28 R11: 473e358cd184502e R12: ffff88017575f4b8 +[ 724.546290] R13: 0000000000000000 R14: ffff880188148400 R15: 0000000000000000 +[ 724.562989] vmx_handle_exit+0x1144/0x17f0 +[ 724.579859] vcpu_enter_guest+0x14a9/0x62e0 +[ 724.673018] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 724.682933] kvm_vcpu_ioctl+0x72b/0x1150 +[ 724.742569] do_vfs_ioctl+0x1de/0x1720 +[ 724.793606] ksys_ioctl+0xa9/0xd0 +[ 724.797074] __x64_sys_ioctl+0x73/0xb0 +[ 724.800988] do_syscall_64+0x1b9/0x820 +[ 724.840073] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 724.845299] RIP: 0033:0x457099 +[ 724.867425] RSP: 002b:00007f0e6289ec78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 724.875170] RAX: ffffffffffffffda RBX: 00007f0e6289f6d4 RCX: 0000000000457099 +[ 724.882446] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000007 +[ 724.889728] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 724.897005] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 724.904285] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/283 b/pkg/report/testdata/linux/report/283 new file mode 100644 index 000000000..817459191 --- /dev/null +++ b/pkg/report/testdata/linux/report/283 @@ -0,0 +1,55 @@ +TITLE: INFO: rcu detected stall in snd_pcm_oss_release + +[ 362.024021] INFO: rcu_sched self-detected stall on CPU +[ 362.029432] 1-....: (124999 ticks this GP) idle=f9a/1/4611686018427387906 softirq=73140/73140 fqs=31238 +[ 362.039293] (t=125000 jiffies g=36742 c=36741 q=633) +[ 362.044549] NMI backtrace for cpu 1 +[ 362.048153] CPU: 1 PID: 17059 Comm: syz-executor4 Not tainted 4.16.0+ #4 +[ 362.054963] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 362.064292] Call Trace: +[ 362.066846] <IRQ> +[ 362.068978] dump_stack+0x1b9/0x294 +[ 362.081704] nmi_cpu_backtrace.cold.4+0x19/0xce +[ 362.091512] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 362.096764] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 362.101928] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 362.111309] check_cpu_stall.isra.61.cold.80+0x36c/0x59a +[ 362.172363] rcu_check_callbacks+0x21b/0xad0 +[ 362.230142] update_process_times+0x2d/0x70 +[ 362.234439] tick_sched_handle+0xa0/0x180 +[ 362.238565] tick_sched_timer+0x42/0x130 +[ 362.242605] __hrtimer_run_queues+0x3e3/0x10a0 +[ 362.290776] hrtimer_interrupt+0x286/0x650 +[ 362.294994] smp_apic_timer_interrupt+0x15d/0x710 +[ 362.326302] apic_timer_interrupt+0xf/0x20 +[ 362.330507] </IRQ> +[ 362.332720] RIP: 0010:__sanitizer_cov_trace_const_cmp1+0x1a/0x20 +[ 362.338837] RSP: 0018:ffff8801c7b5ede0 EFLAGS: 00000293 ORIG_RAX: ffffffffffffff13 +[ 362.346520] RAX: 0000000000000000 RBX: ffff8801ce12c8c0 RCX: ffffffff85a1dc9d +[ 362.353762] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000001 +[ 362.361011] RBP: ffff8801c7b5ede0 R08: ffff8801c3d94300 R09: 0000000000000006 +[ 362.368261] R10: ffff8801c3d94300 R11: 0000000000000000 R12: 0000000000000000 +[ 362.375505] R13: 0000000000000000 R14: ffff8801ceaaed80 R15: ffff8801ce12cc18 +[ 362.387226] snd_pcm_oss_prepare+0x7d/0x1b0 +[ 362.391529] snd_pcm_oss_write3+0x1a7/0x220 +[ 362.395826] snd_pcm_oss_write2+0x34c/0x460 +[ 362.408634] snd_pcm_oss_sync1+0x332/0x5a0 +[ 362.448533] snd_pcm_oss_sync.isra.29+0x790/0x980 +[ 362.466072] snd_pcm_oss_release+0x214/0x290 +[ 362.475452] __fput+0x34d/0x890 +[ 362.490736] ____fput+0x15/0x20 +[ 362.493989] task_work_run+0x1e4/0x290 +[ 362.512311] do_exit+0x1aee/0x2730 +[ 362.635785] do_group_exit+0x16f/0x430 +[ 362.657106] get_signal+0x886/0x1960 +[ 362.717833] do_signal+0x98/0x2040 +[ 362.764335] exit_to_usermode_loop+0x28a/0x310 +[ 362.781265] do_syscall_64+0x792/0x9d0 +[ 362.813438] entry_SYSCALL_64_after_hwframe+0x42/0xb7 +[ 362.818600] RIP: 0033:0x455259 +[ 362.821763] RSP: 002b:00007fb730463ce8 EFLAGS: 00000246 ORIG_RAX: 00000000000000ca +[ 362.829446] RAX: fffffffffffffe00 RBX: 000000000072bf80 RCX: 0000000000455259 +[ 362.836692] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 000000000072bf80 +[ 362.843936] RBP: 000000000072bf80 R08: 0000000000000000 R09: 000000000072bf58 +[ 362.851177] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000 +[ 362.858424] R13: 00007ffc071d643f R14: 00007fb7304649c0 R15: 0000000000000001 diff --git a/pkg/report/testdata/linux/report/284 b/pkg/report/testdata/linux/report/284 new file mode 100644 index 000000000..9eec50d2b --- /dev/null +++ b/pkg/report/testdata/linux/report/284 @@ -0,0 +1,51 @@ +TITLE: INFO: rcu detected stall in llcp_sock_sendmsg + +[ 1098.520844] INFO: rcu_sched self-detected stall on CPU +[ 1098.520919] 1-....: (20918 ticks this GP) idle=55a/1/4611686018427387906 softirq=11347/11347 fqs=20240 +[ 1098.520943] (t=125005 jiffies g=5572 c=5571 q=149) +[ 1098.520955] NMI backtrace for cpu 1 +[ 1098.520964] CPU: 1 PID: 4811 Comm: syz-executor0 Not tainted 4.18.0-rc1+ #115 +[ 1098.520969] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1098.521036] Call Trace: +[ 1098.521040] <IRQ> +[ 1098.521119] dump_stack+0x1c9/0x2b4 +[ 1098.521173] nmi_cpu_backtrace.cold.4+0x19/0xce +[ 1098.521197] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 1098.521209] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 1098.521220] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 1098.521245] check_cpu_stall.isra.60.cold.78+0x36c/0x5a6 +[ 1098.521438] rcu_check_callbacks+0x23f/0xcd0 +[ 1098.521611] update_process_times+0x2d/0x70 +[ 1098.521623] tick_sched_handle+0x9f/0x180 +[ 1098.521635] tick_sched_timer+0x45/0x130 +[ 1098.521645] __hrtimer_run_queues+0x3eb/0x10c0 +[ 1098.521777] hrtimer_interrupt+0x2f3/0x750 +[ 1098.521808] smp_apic_timer_interrupt+0x165/0x730 +[ 1098.521901] apic_timer_interrupt+0xf/0x20 +[ 1098.521906] </IRQ> +[ 1098.521915] RIP: 0010:console_unlock+0xc84/0x10b0 +[ 1098.522104] RSP: 0018:ffff8801aab0f358 EFLAGS: 00000293 ORIG_RAX: ffffffffffffff13 +[ 1098.522115] RAX: ffff8801aa2802c0 RBX: 0000000000000200 RCX: 1ffff10035450163 +[ 1098.522121] RDX: 0000000000000000 RSI: ffffffff8162b8fb RDI: 0000000000000293 +[ 1098.522127] RBP: ffff8801aab0f4c0 R08: ffff8801aa280af8 R09: 0000000000000006 +[ 1098.522132] R10: ffff8801aa2802c0 R11: 0000000000000000 R12: 0000000000000000 +[ 1098.522138] R13: ffffffff84ea9880 R14: 0000000000000001 R15: dffffc0000000000 +[ 1098.522273] vprintk_emit+0x6c6/0xdf0 +[ 1098.522366] vprintk_default+0x28/0x30 +[ 1098.522375] vprintk_func+0x7a/0xe7 +[ 1098.522385] printk+0xa7/0xcf +[ 1098.522435] nfc_llcp_send_ui_frame.cold.9+0x18/0x1f +[ 1098.522507] llcp_sock_sendmsg+0x278/0x350 +[ 1098.522528] sock_sendmsg+0xd5/0x120 +[ 1098.522539] ___sys_sendmsg+0x51d/0x930 +[ 1098.522667] __sys_sendmmsg+0x240/0x6f0 +[ 1098.522794] __x64_sys_sendmmsg+0x9d/0x100 +[ 1098.522806] do_syscall_64+0x1b9/0x820 +[ 1098.522882] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 1098.522891] RIP: 0033:0x455a99 +[ 1098.523077] RSP: 002b:00007ffec82fb2b8 EFLAGS: 00000246 ORIG_RAX: 0000000000000133 +[ 1098.523087] RAX: ffffffffffffffda RBX: 0000000001992914 RCX: 0000000000455a99 +[ 1098.523093] RDX: 04924924924926b2 RSI: 00000000200026c0 RDI: 0000000000000004 +[ 1098.523099] RBP: 000000000072bea0 R08: 0000000000000000 R09: 0000000000000000 +[ 1098.523104] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 1098.523110] R13: 00000000004c0b44 R14: 00000000004d0980 R15: 0000000000000001 diff --git a/pkg/report/testdata/linux/report/285 b/pkg/report/testdata/linux/report/285 new file mode 100644 index 000000000..213eccf77 --- /dev/null +++ b/pkg/report/testdata/linux/report/285 @@ -0,0 +1,64 @@ +TITLE: INFO: rcu detected stall in sctp_generate_heartbeat_event + +[ 396.651023] INFO: rcu_sched self-detected stall on CPU +[ 396.656449] 0-...!: (120232 ticks this GP) idle=3da/1/4611686018427387906 softirq=85066/85067 fqs=87 +[ 396.666018] (t=125000 jiffies g=45941 c=45940 q=632561) +[ 396.671551] rcu_sched kthread starved for 96651 jiffies! g45941 c45940 f0x2 RCU_GP_WAIT_FQS(3) ->state=0x0 ->cpu=1 +[ 396.682013] RCU grace-period kthread stack dump: +[ 396.686747] rcu_sched R running task 22584 9 2 0x80000000 +[ 396.693925] Call Trace: +[ 396.696504] __schedule+0x801/0x1e30 +[ 396.721528] schedule+0xef/0x430 +[ 396.760095] schedule_timeout+0x138/0x240 +[ 396.773155] rcu_gp_kthread+0x6b5/0x1940 +[ 396.902194] kthread+0x345/0x410 +[ 396.914410] ret_from_fork+0x3a/0x50 +[ 396.918117] NMI backtrace for cpu 0 +[ 396.921726] CPU: 0 PID: 8 Comm: ksoftirqd/0 Not tainted 4.17.0-rc4+ #48 +[ 396.928465] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 396.937797] Call Trace: +[ 396.940361] <IRQ> +[ 396.942495] dump_stack+0x1b9/0x294 +[ 396.955257] nmi_cpu_backtrace.cold.4+0x19/0xce +[ 396.965085] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 396.970349] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 396.975523] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 396.984915] check_cpu_stall.isra.61.cold.80+0x36c/0x59a +[ 397.045471] rcu_check_callbacks+0x21b/0xad0 +[ 397.099148] update_process_times+0x2d/0x70 +[ 397.103455] tick_sched_handle+0x9f/0x180 +[ 397.107586] tick_sched_timer+0x45/0x130 +[ 397.111633] __hrtimer_run_queues+0x3e3/0x10a0 +[ 397.164906] hrtimer_interrupt+0x2f3/0x750 +[ 397.169132] smp_apic_timer_interrupt+0x15d/0x710 +[ 397.200493] apic_timer_interrupt+0xf/0x20 +[ 397.204703] </IRQ> +[ 397.206926] RIP: 0010:unwind_next_frame.part.7+0x8b/0x9c0 +[ 397.212439] RSP: 0018:ffff8801d9aaebb0 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 397.220131] RAX: 1ffff1003b355da3 RBX: ffff8801d9aaecc8 RCX: 0000000000000007 +[ 397.227380] RDX: dffffc0000000000 RSI: ffffffff86e8b04f RDI: ffff8801d9aaecc8 +[ 397.234633] RBP: ffff8801d9aaeca0 R08: ffff8801d9aaed00 R09: ffff8801d9a9c200 +[ 397.241882] R10: ffffed003b355da3 R11: ffff8801d9aaed1f R12: 1ffff1003b355d7b +[ 397.249135] R13: 0000000000000000 R14: ffff8801d9a9c200 R15: ffff8801d9aaed18 +[ 397.284735] unwind_next_frame+0x3e/0x50 +[ 397.288784] __save_stack_trace+0x6e/0xd0 +[ 397.297403] save_stack_trace+0x1a/0x20 +[ 397.301360] save_stack+0x43/0xd0 +[ 397.415393] kasan_kmalloc+0xc4/0xe0 +[ 397.419094] kasan_slab_alloc+0x12/0x20 +[ 397.423094] kmem_cache_alloc+0x12e/0x760 +[ 397.427240] sctp_chunkify+0xce/0x400 +[ 397.455406] _sctp_make_chunk+0x157/0x280 +[ 397.459538] sctp_make_heartbeat+0x8f/0x430 +[ 397.483422] sctp_sf_heartbeat.isra.23+0x26/0x180 +[ 397.488250] sctp_sf_sendbeat_8_3+0x38e/0x550 +[ 397.492731] sctp_do_sm+0x1ab/0x7160 +[ 397.565731] sctp_generate_heartbeat_event+0x218/0x450 +[ 397.571003] call_timer_fn+0x230/0x940 +[ 397.617386] __run_timers+0x79e/0xc50 +[ 397.668620] run_timer_softirq+0x4c/0x70 +[ 397.672666] __do_softirq+0x2e0/0xaf5 +[ 397.707640] run_ksoftirqd+0x86/0x100 +[ 397.711424] smpboot_thread_fn+0x417/0x870 +[ 397.729086] kthread+0x345/0x410 +[ 397.739836] ret_from_fork+0x3a/0x50 diff --git a/pkg/report/testdata/linux/report/286 b/pkg/report/testdata/linux/report/286 new file mode 100644 index 000000000..1e625a25d --- /dev/null +++ b/pkg/report/testdata/linux/report/286 @@ -0,0 +1,90 @@ +TITLE: INFO: rcu detected stall in sctp_generate_heartbeat_event + +[ 486.569033] INFO: rcu_sched self-detected stall on CPU +[ 486.574490] 0-...!: (124998 ticks this GP) idle=0be/1/4611686018427387908 softirq=15234/15234 fqs=59 +[ 486.584061] (t=125000 jiffies g=7610 c=7609 q=351640) +[ 486.589425] rcu_sched kthread starved for 124739 jiffies! g7610 c7609 f0x2 RCU_GP_WAIT_FQS(3) ->state=0x0 ->cpu=1 +[ 486.599806] RCU grace-period kthread stack dump: +[ 486.604550] rcu_sched R running task 23896 9 2 0x80000000 +[ 486.611759] Call Trace: +[ 486.614337] __schedule+0x801/0x1e30 +[ 486.639334] schedule+0xef/0x430 +[ 486.678051] schedule_timeout+0x138/0x240 +[ 486.690987] rcu_gp_kthread+0x6b5/0x1940 +[ 486.824473] kthread+0x345/0x410 +[ 486.836697] ret_from_fork+0x3a/0x50 +[ 486.840421] NMI backtrace for cpu 0 +[ 486.844038] CPU: 0 PID: 6381 Comm: sh Not tainted 4.17.0-rc5+ #58 +[ 486.850257] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 486.859596] Call Trace: +[ 486.862174] <IRQ> +[ 486.864315] dump_stack+0x1b9/0x294 +[ 486.877074] nmi_cpu_backtrace.cold.4+0x19/0xce +[ 486.886903] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 486.892167] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 486.897347] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 486.906765] check_cpu_stall.isra.61.cold.80+0x36c/0x59a +[ 486.967369] rcu_check_callbacks+0x21b/0xad0 +[ 487.021142] update_process_times+0x2d/0x70 +[ 487.025452] tick_sched_handle+0x9f/0x180 +[ 487.029583] tick_sched_timer+0x45/0x130 +[ 487.033630] __hrtimer_run_queues+0x3e3/0x10a0 +[ 487.074515] hrtimer_interrupt+0x2f3/0x750 +[ 487.078754] smp_apic_timer_interrupt+0x15d/0x710 +[ 487.098448] apic_timer_interrupt+0xf/0x20 +[ 487.102683] RIP: 0010:lock_acquire+0x257/0x520 +[ 487.107246] RSP: 0000:ffff8801dae06b60 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 +[ 487.114937] RAX: dffffc0000000000 RBX: 1ffff1003b5c0d71 RCX: 0000000000000000 +[ 487.122190] RDX: 1ffffffff11a30e5 RSI: ffff8801c7b54c38 RDI: 0000000000000286 +[ 487.129440] RBP: ffff8801dae06c50 R08: 0000000000000008 R09: 0000000000000003 +[ 487.136692] R10: ffff8801c7b54cb0 R11: ffff8801c7b54400 R12: ffff8801c7b54400 +[ 487.143948] R13: 0000000000000002 R14: 0000000000000000 R15: 0000000000000000 +[ 487.173035] is_bpf_text_address+0x3b/0x170 +[ 487.181997] kernel_text_address+0x79/0xf0 +[ 487.190350] __kernel_text_address+0xd/0x40 +[ 487.194668] unwind_get_return_address+0x61/0xa0 +[ 487.199429] __save_stack_trace+0x7e/0xd0 +[ 487.203571] save_stack_trace+0x1a/0x20 +[ 487.207529] save_stack+0x43/0xd0 +[ 487.356403] kasan_kmalloc+0xc4/0xe0 +[ 487.360107] kasan_slab_alloc+0x12/0x20 +[ 487.364754] kmem_cache_alloc_node+0x131/0x780 +[ 487.383127] __alloc_skb+0x111/0x780 +[ 487.426126] _sctp_make_chunk+0x58/0x280 +[ 487.430174] sctp_make_heartbeat+0x8f/0x430 +[ 487.459807] sctp_sf_heartbeat.isra.23+0x26/0x180 +[ 487.464634] sctp_sf_sendbeat_8_3+0x38e/0x550 +[ 487.469117] sctp_do_sm+0x1ab/0x7160 +[ 487.544210] sctp_generate_heartbeat_event+0x218/0x450 +[ 487.549480] call_timer_fn+0x230/0x940 +[ 487.604501] __run_timers+0x79e/0xc50 +[ 487.647862] run_timer_softirq+0x4c/0x70 +[ 487.651907] __do_softirq+0x2e0/0xaf5 +[ 487.696662] irq_exit+0x1d1/0x200 +[ 487.700103] smp_apic_timer_interrupt+0x17e/0x710 +[ 487.731472] apic_timer_interrupt+0xf/0x20 +[ 487.735686] </IRQ> +[ 487.737908] RIP: 0010:rcu_is_watching+0xc4/0x140 +[ 487.742651] RSP: 0000:ffff8801c957f398 EFLAGS: 00000a06 ORIG_RAX: ffffffffffffff13 +[ 487.750347] RAX: 00000000000130be RBX: 1ffff100392afe74 RCX: 1ffff100392afe78 +[ 487.757598] RDX: 0000000000000004 RSI: 0000000000000004 RDI: ffff8801dae23610 +[ 487.764850] RBP: ffff8801c957f428 R08: ffffed003b5c46c3 R09: ffffed003b5c46c2 +[ 487.772101] R10: ffffed003b5c46c2 R11: ffff8801dae23613 R12: ffff8801c957f3c0 +[ 487.779353] R13: ffff8801dae23610 R14: ffff8801c957f400 R15: dffffc0000000000 +[ 487.804394] rcu_read_lock_sched_held+0x8d/0x120 +[ 487.809139] kmem_cache_alloc+0x5fa/0x760 +[ 487.821900] __anon_vma_prepare+0x3b3/0x700 +[ 487.838290] wp_page_copy+0xdf1/0x1440 +[ 487.873330] do_wp_page+0x425/0x1990 +[ 487.929184] __handle_mm_fault+0x2996/0x4310 +[ 487.958511] handle_mm_fault+0x53a/0xc70 +[ 487.970662] __do_page_fault+0x60b/0xe40 +[ 487.982996] do_page_fault+0xee/0x8a7 +[ 488.035994] page_fault+0x1e/0x30 +[ 488.039438] RIP: 0033:0x7f4232a6d1fb +[ 488.043133] RSP: 002b:00007fff2aeebd28 EFLAGS: 00010246 +[ 488.048483] RAX: 000000000000000d RBX: 00000000756e6547 RCX: 000000006c65746e +[ 488.055740] RDX: 0000000049656e69 RSI: 0000000000000025 RDI: 0000000000000002 +[ 488.063019] RBP: 00007fff2aeebe20 R08: 0000000000000001 R09: 000000000000002f +[ 488.070280] R10: 00007f4232a6caf0 R11: 00007fff2aeebb70 R12: 00007f4232a4e000 +[ 488.077543] R13: 00007f4232dd5038 R14: 00007f4232a6cac0 R15: 00007f4232dd5d98 diff --git a/pkg/report/testdata/linux/report/287 b/pkg/report/testdata/linux/report/287 new file mode 100644 index 000000000..962845d9f --- /dev/null +++ b/pkg/report/testdata/linux/report/287 @@ -0,0 +1,53 @@ +TITLE: INFO: rcu detected stall in br_handle_frame + +[ 268.109997] bridge0: received packet on bridge_slave_0 with own address as source address +[ 268.110510] bridge0: received packet on bridge_slave_0 with own address as source address +[ 268.111039] bridge0: received packet on bridge_slave_0 with own address as source address +[ 268.638278] INFO: rcu_sched self-detected stall on CPU +[ 268.643724] 1-...: (10500 ticks this GP) idle=7fb/140000000000001/0 softirq=7363/7366 fqs=2223 +[ 268.652635] (t=10500 jiffies g=2314 c=2313 q=216) +[ 268.657826] Task dump for CPU 1: +[ 268.661186] kworker/u4:2 R running task 27384 2030 2 0x00000008 +[ 268.668655] Workqueue: netns cleanup_net +[ 268.672841] ffff8801ca698000 ffff8801db7072e0 ffffffff813eb3e6 1ffff1003b6e3ebf +[ 268.680895] ffff8801db71ecc0 0000000000000001 dffffc0000000000 ffffffff832dcec0 +[ 268.688951] ffffffff832dcf44 ffff8801db7072f0 ffffffff813eb402 ffff8801db707310 +[ 268.697003] Call Trace: +[ 268.699573] <IRQ> [<ffffffff813eb3e6>] _sched_show_task+0x2d0/0x2dc +[ 268.706289] [<ffffffff813eb402>] sched_show_task+0x10/0x12 +[ 268.711995] [<ffffffff813eb53d>] dump_cpu_task+0x7e/0x83 +[ 268.717534] [<ffffffff813f1168>] rcu_dump_cpu_stacks+0x155/0x169 +[ 268.723767] [<ffffffff813f1b4f>] rcu_check_callbacks.cold.83+0x5f5/0xc05 +[ 268.730700] [<ffffffff8125d08f>] update_process_times+0x3f/0x70 +[ 268.736853] [<ffffffff81286f8a>] tick_sched_handle.isra.17+0x5a/0xf0 +[ 268.743427] [<ffffffff812877a7>] tick_sched_timer+0x77/0x130 +[ 268.755533] [<ffffffff8126021c>] __hrtimer_run_queues+0x36c/0xda0 +[ 268.786608] [<ffffffff8126165b>] hrtimer_interrupt+0x18b/0x3f0 +[ 268.792660] [<ffffffff810911aa>] local_apic_timer_interrupt+0x7a/0xa0 +[ 268.799325] [<ffffffff82a608c1>] smp_apic_timer_interrupt+0x81/0xb0 +[ 268.805809] [<ffffffff82a5ecc0>] apic_timer_interrupt+0xa0/0xb0 +[ 268.823881] [<ffffffff822e04aa>] lockdep_rtnl_is_held+0x1a/0x30 +[ 268.830020] [<ffffffff8297ead9>] br_validate_ipv6+0x779/0xcf0 +[ 268.854769] [<ffffffff8297f0d5>] br_nf_pre_routing_ipv6+0x85/0x520 +[ 268.884956] [<ffffffff8297cd22>] br_nf_pre_routing+0x8c2/0x1040 +[ 268.903028] [<ffffffff823e2b77>] nf_iterate+0x187/0x210 +[ 268.908474] [<ffffffff823e2db5>] nf_hook_slow+0x1b5/0x330 +[ 268.931204] [<ffffffff8294e062>] br_handle_frame+0x732/0xca0 +[ 268.957406] [<ffffffff822ab8d4>] __netif_receive_skb_core+0x7b4/0x27e0 +[ 268.982260] [<ffffffff822ad960>] __netif_receive_skb+0x60/0x1c0 +[ 268.988398] [<ffffffff822b1c6d>] process_backlog+0x21d/0x6a0 +[ 269.000323] [<ffffffff822aec29>] net_rx_action+0x2d9/0xc50 +[ 269.024455] [<ffffffff82a610e1>] __do_softirq+0x231/0x9f2 +[ 269.036557] [<ffffffff82a5fb3c>] do_softirq_own_stack+0x1c/0x30 +[ 269.042690] <EOI> [<ffffffff8111a2a9>] do_softirq.part.17+0x59/0x60 +[ 269.049401] [<ffffffff8111a369>] __local_bh_enable_ip+0xb9/0xc0 +[ 269.055544] [<ffffffff8240181d>] nf_ct_iterate_cleanup+0x10d/0x520 +[ 269.068606] [<ffffffff824160b3>] nf_ct_l3proto_pernet_unregister+0xb3/0xe0 +[ 269.081669] [<ffffffff82779601>] ipv4_net_exit+0x21/0x60 +[ 269.087203] [<ffffffff8228ae55>] ops_exit_list.isra.6+0xb5/0x160 +[ 269.093438] [<ffffffff8228ddae>] cleanup_net+0x31e/0x5e0 +[ 269.110339] [<ffffffff8115eba9>] process_one_work+0x7c9/0x14f0 +[ 269.128745] [<ffffffff8115f9ae>] worker_thread+0xde/0xfc0 +[ 269.141289] [<ffffffff8116dfb2>] kthread+0x272/0x310 +[ 269.171975] [<ffffffff82a5e1fe>] ret_from_fork+0x4e/0x80 +[ 273.109547] net_ratelimit: 10606 callbacks suppressed diff --git a/pkg/report/testdata/linux/report/288 b/pkg/report/testdata/linux/report/288 new file mode 100644 index 000000000..cde967fb2 --- /dev/null +++ b/pkg/report/testdata/linux/report/288 @@ -0,0 +1,89 @@ +TITLE: BUG: soft lockup in kvm_vcpu_release + +[ 1342.342232] watchdog: BUG: soft lockup - CPU#0 stuck for 123s! [syz-executor0:8009] +[ 1342.350104] Modules linked in: +[ 1342.353311] irq event stamp: 32790 +[ 1342.356862] hardirqs last enabled at (32789): [<ffffffff81007d91>] trace_hardirqs_on_thunk+0x1a/0x1c +[ 1342.366247] hardirqs last disabled at (32790): [<ffffffff81007dad>] trace_hardirqs_off_thunk+0x1a/0x1c +[ 1342.375718] softirqs last enabled at (4938): [<ffffffff87e007ba>] __do_softirq+0x7ba/0xad8 +[ 1342.384222] softirqs last disabled at (4873): [<ffffffff814ab9ff>] irq_exit+0x17f/0x1c0 +[ 1342.393426] CPU: 0 PID: 8009 Comm: syz-executor0 Not tainted 4.19.0-rc2+ #230 +[ 1342.400705] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1342.410083] RIP: 0010:__sanitizer_cov_trace_pc+0x20/0x50 +[ 1342.434452] RSP: 0018:ffff8801b9257390 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 1342.442171] RAX: ffff8801940f4580 RBX: ffff8801b9257418 RCX: ffffffff817284a9 +[ 1342.449440] RDX: 0000000000000000 RSI: ffffffff81728458 RDI: 0000000000000005 +[ 1342.456711] RBP: ffff8801b9257390 R08: ffff8801940f4580 R09: ffffed003b5e5ba0 +[ 1342.463980] R10: ffffed003b5e5ba0 R11: ffff8801daf2dd07 R12: ffffed003724ae90 +[ 1342.471269] R13: 1ffff1003724ae7c R14: dffffc0000000000 R15: 0000000000000001 +[ 1342.478545] FS: 000000000124c940(0000) GS:ffff8801dae00000(0000) knlGS:0000000000000000 +[ 1342.486771] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 1342.492651] CR2: 0000000020fea800 CR3: 00000001c17e1000 CR4: 00000000001426f0 +[ 1342.499927] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 1342.507203] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 1342.514471] Call Trace: +[ 1342.517069] smp_call_function_single+0x258/0x660 +[ 1342.546284] vmx_vcpu_load+0x8a7/0x1030 +[ 1342.587166] kvm_arch_vcpu_load+0x247/0x970 +[ 1342.606789] vcpu_load+0x35/0x70 +[ 1342.610160] kvm_arch_destroy_vm+0x1f9/0x7c0 +[ 1342.644514] kvm_put_kvm+0x6c8/0xff0 +[ 1342.676490] kvm_vcpu_release+0x7b/0xa0 +[ 1342.680470] __fput+0x385/0xa30 +[ 1342.714127] ____fput+0x15/0x20 +[ 1342.717411] task_work_run+0x1e8/0x2a0 +[ 1342.733866] exit_to_usermode_loop+0x318/0x380 +[ 1342.748873] do_syscall_64+0x6be/0x820 +[ 1342.782821] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 1342.788012] RIP: 0033:0x410c51 +[ 1342.810143] RSP: 002b:0000000000a3fdc0 EFLAGS: 00000293 ORIG_RAX: 0000000000000003 +[ 1342.817870] RAX: 0000000000000000 RBX: 0000000000000006 RCX: 0000000000410c51 +[ 1342.825144] RDX: 0000000000000000 RSI: 0000000000730f10 RDI: 0000000000000005 +[ 1342.832419] RBP: 0000000000000000 R08: ffffffffffffffff R09: ffffffffffffffff +[ 1342.839694] R10: 0000000000a3fcf0 R11: 0000000000000293 R12: 0000000000000007 +[ 1342.846968] R13: 0000000000123610 R14: 0000000000000323 R15: badc0ffeebadface +[ 1342.854275] Sending NMI from CPU 0 to CPUs 1: +[ 1342.860797] NMI backtrace for cpu 1 +[ 1342.860804] CPU: 1 PID: 8029 Comm: syz-executor4 Not tainted 4.19.0-rc2+ #230 +[ 1342.860811] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1342.860816] RIP: 0010:vmx_read_guest_seg_ar+0x1ff/0x270 +[ 1342.860834] RSP: 0018:ffff8801c020edf0 EFLAGS: 00000807 +[ 1342.860844] RAX: dffffc0000000000 RBX: 0000000000000002 RCX: ffffffff811c33b0 +[ 1342.860850] RDX: 0000000000000000 RSI: ffffffff811c33be RDI: ffff8801c757eafc +[ 1342.860856] RBP: ffff8801c020ee18 R08: ffff8801c88a04c0 R09: fffff520026ac047 +[ 1342.860862] R10: fffff520026ac047 R11: ffffc9001356023b R12: ffff8801c75790c0 +[ 1342.860868] R13: 0000000000000800 R14: 0000000000000800 R15: 0000000000100000 +[ 1342.860874] FS: 00007fae23f34700(0000) GS:ffff8801daf00000(0000) knlGS:0000000000000000 +[ 1342.860879] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 1342.860885] CR2: 0000001b31d25000 CR3: 00000001bf9be000 CR4: 00000000001426e0 +[ 1342.860891] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 1342.860897] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 1342.860901] Call Trace: +[ 1342.860905] vmx_get_cpl+0x5b/0x90 +[ 1342.860909] kvm_arch_vcpu_put+0x9b/0x420 +[ 1342.860913] kvm_sched_out+0x91/0xb0 +[ 1342.860917] __schedule+0xf8c/0x1ed0 +[ 1342.860958] preempt_schedule_notrace+0x70/0x130 +[ 1342.860962] ___preempt_schedule_notrace+0x16/0x31 +[ 1342.860967] rcu_is_watching+0x23/0x30 +[ 1342.860971] rcu_read_lock+0x43/0x70 +[ 1342.860975] get_mem_cgroup_from_mm+0x8c/0x440 +[ 1342.861014] memcg_kmem_get_cache+0x1fc/0x9d0 +[ 1342.861036] kmem_cache_alloc+0x193/0x730 +[ 1342.861050] mmu_topup_memory_caches+0x2ec/0x390 +[ 1342.861054] kvm_mmu_load+0x21/0xfa0 +[ 1342.861071] vcpu_enter_guest+0x3dee/0x62e0 +[ 1342.861143] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 1342.861152] kvm_vcpu_ioctl+0x72b/0x1150 +[ 1342.861211] do_vfs_ioctl+0x1de/0x1720 +[ 1342.861272] ksys_ioctl+0xa9/0xd0 +[ 1342.861276] __x64_sys_ioctl+0x73/0xb0 +[ 1342.861280] do_syscall_64+0x1b9/0x820 +[ 1342.861318] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 1342.861322] RIP: 0033:0x457099 +[ 1342.861339] RSP: 002b:00007fae23f33c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 1342.861349] RAX: ffffffffffffffda RBX: 00007fae23f346d4 RCX: 0000000000457099 +[ 1342.861355] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000005 +[ 1342.861361] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 1342.861367] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 1342.861373] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/289 b/pkg/report/testdata/linux/report/289 new file mode 100644 index 000000000..febcd4d79 --- /dev/null +++ b/pkg/report/testdata/linux/report/289 @@ -0,0 +1,41 @@ +TITLE: INFO: rcu detected stall in ext4_filemap_fault + +[ 2355.201776] INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 2355.201784] Tasks blocked on level-0 rcu_node (CPUs 0-1): P2270 +[ 2355.201792] (detected by 0, t=10502 jiffies, g=98916, c=98915, q=146) +[ 2355.201808] syz-fuzzer R running task 25680 2270 2264 0x00000000 +[ 2355.201818] 00000000000008d8 16c3269289c67388 ffff8801db607c20 ffffffff8138d924 +[ 2355.201826] ffff8801caf9df00 ffffffff82ea86c0 0000000000000092 dffffc0000000000 +[ 2355.201835] ffff8801caf9e308 ffff8801db607c58 ffffffff8139811e 0000000000018263 +[ 2355.201836] Call Trace: +[ 2355.201852] <IRQ> [<ffffffff8138d924>] sched_show_task+0x378/0x387 +[ 2355.201860] [<ffffffff8139811e>] rcu_print_detail_task_stall_rnp+0xbd/0xf9 +[ 2355.201868] [<ffffffff8139910c>] rcu_check_callbacks.cold.67+0xa85/0xd20 +[ 2355.201876] [<ffffffff81251b3a>] update_process_times+0x3a/0x70 +[ 2355.201885] [<ffffffff8127e0fa>] tick_sched_handle.isra.6+0x4a/0xf0 +[ 2355.201892] [<ffffffff8127e216>] tick_sched_timer+0x76/0x130 +[ 2355.201906] [<ffffffff81253bb0>] __hrtimer_run_queues+0x390/0xfc0 +[ 2355.201956] [<ffffffff81256831>] hrtimer_interrupt+0x1b1/0x430 +[ 2355.201964] [<ffffffff810897c4>] local_apic_timer_interrupt+0x74/0xa0 +[ 2355.201971] [<ffffffff82692b7c>] smp_apic_timer_interrupt+0x7c/0xa0 +[ 2355.201978] [<ffffffff82691edd>] apic_timer_interrupt+0x9d/0xb0 +[ 2355.201987] <EOI> [<ffffffff81214699>] ? console_unlock+0x659/0xa10 +[ 2355.202000] [<ffffffff81214e45>] vprintk_emit+0x3f5/0x830 +[ 2355.202006] [<ffffffff812152a8>] vprintk+0x28/0x30 +[ 2355.202013] [<ffffffff812152cd>] vprintk_default+0x1d/0x30 +[ 2355.202019] [<ffffffff8139632a>] printk+0xaf/0xd7 +[ 2355.202042] [<ffffffff8146763e>] dump_header.isra.4+0x617/0x6b5 +[ 2355.202050] [<ffffffff814676f7>] oom_kill_process.cold.9+0x1b/0x9b6 +[ 2355.202065] [<ffffffff813a9d8e>] out_of_memory+0x48e/0xab0 +[ 2355.202092] [<ffffffff813b600a>] __alloc_pages_nodemask+0x122a/0x1430 +[ 2355.202166] [<ffffffff813a5a22>] filemap_fault+0x802/0xbe0 +[ 2355.202173] [<ffffffff81634781>] ext4_filemap_fault+0x71/0xa0 +[ 2355.202181] [<ffffffff8140dee7>] __do_fault+0x1d7/0x360 +[ 2355.202203] [<ffffffff8141abd9>] handle_mm_fault+0x1cd9/0x2f30 +[ 2355.202231] [<ffffffff810a9571>] __do_page_fault+0x291/0x7e0 +[ 2355.202238] [<ffffffff810a9b17>] do_page_fault+0x27/0x30 +[ 2355.202244] [<ffffffff82691775>] page_fault+0x25/0x30 +[ 2355.202257] syz-fuzzer R running task 25680 2270 2264 0x00000000 +[ 2355.202266] 00000000000008d8 16c3269289c67388 ffff8801db607c20 ffffffff8138d924 +[ 2355.202274] ffff8801caf9df00 ffffffff82ea86c0 0000000000000092 dffffc0000000000 +[ 2355.202283] ffff8801caf9e308 ffff8801db607c58 ffffffff8139811e ffffffff82ea8940 diff --git a/pkg/report/testdata/linux/report/290 b/pkg/report/testdata/linux/report/290 new file mode 100644 index 000000000..f5784c806 --- /dev/null +++ b/pkg/report/testdata/linux/report/290 @@ -0,0 +1,47 @@ +TITLE: INFO: rcu detected stall in ipv6_rcv + +[ 757.882665] INFO: rcu_preempt self-detected stall on CPU +[ 757.888422] 0-...: (1 GPs behind) idle=c37/140000000000001/0 softirq=7357/7358 fqs=12499 +[ 757.896793] (t=12500 jiffies g=1208 c=1207 q=83) +[ 757.901825] Task dump for CPU 0: +[ 757.905161] syz-executor0 R running task 28096 4312 3955 0x2002000c +[ 757.912583] 0000000000000f73 f9a9a8cb61184120 ffff8801db206ff0 ffffffff8140c8fc +[ 757.920592] ffff8801db21f4c0 0000000000000000 dffffc0000000000 ffffffff844bef00 +[ 757.928622] ffffffff844bef84 ffff8801db207010 ffffffff8140cb87 ffffffff844bef48 +[ 757.936617] Call Trace: +[ 757.939184] <IRQ> [<ffffffff8140c8fc>] sched_show_task+0x2cb/0x2d6 +[ 757.945782] [<ffffffff8140cb87>] dump_cpu_task+0x79/0x7e +[ 757.951308] [<ffffffff8141704d>] rcu_dump_cpu_stacks+0x150/0x164 +[ 757.957513] [<ffffffff81417c4a>] rcu_check_callbacks.cold.75+0x5c3/0xd20 +[ 757.970200] [<ffffffff8129a7da>] update_process_times+0x3a/0x70 +[ 757.976320] [<ffffffff812c5195>] tick_sched_handle.isra.15+0x55/0xf0 +[ 757.982881] [<ffffffff812c5822>] tick_sched_timer+0x72/0x120 +[ 757.994948] [<ffffffff8129dd4d>] __hrtimer_run_queues+0x3ad/0x1000 +[ 758.025989] [<ffffffff8129f4c1>] hrtimer_interrupt+0x1b1/0x430 +[ 758.032028] [<ffffffff810ad284>] local_apic_timer_interrupt+0x74/0xa0 +[ 758.038752] [<ffffffff838c534c>] smp_apic_timer_interrupt+0x7c/0xa0 +[ 758.045218] [<ffffffff838c4290>] apic_timer_interrupt+0xa0/0xb0 +[ 758.063861] [<ffffffff833c9b39>] __xfrm_decode_session+0x69/0x100 +[ 758.070171] [<ffffffff834a3d9e>] icmpv6_route_lookup+0x2ce/0x440 +[ 758.088109] [<ffffffff834a5b99>] icmp6_send+0xee9/0x1b80 +[ 758.152555] [<ffffffff8355b491>] icmpv6_send+0xb1/0x1b0 +[ 758.164964] [<ffffffff8346309d>] ip6_pkt_drop+0x16d/0x430 +[ 758.170569] [<ffffffff8346337c>] ip6_pkt_discard+0x1c/0x20 +[ 758.176265] [<ffffffff834360bd>] ip6_rcv_finish+0x13d/0x640 +[ 758.187979] [<ffffffff83438bdb>] ipv6_rcv+0x10cb/0x1cd0 +[ 758.217187] [<ffffffff82f7eb06>] __netif_receive_skb_core+0x12d6/0x2940 +[ 758.237196] [<ffffffff82f801cb>] __netif_receive_skb+0x5b/0x1b0 +[ 758.243331] [<ffffffff82f84756>] process_backlog+0x216/0x6a0 +[ 758.255225] [<ffffffff82f81592>] net_rx_action+0x3a2/0xdb0 +[ 758.274043] [<ffffffff838c5bec>] __do_softirq+0x22c/0xa1a +[ 758.286381] [<ffffffff838c399c>] do_softirq_own_stack+0x1c/0x30 +[ 758.292494] <EOI> [<ffffffff8113d9d4>] do_softirq.part.16+0x54/0x60 +[ 758.299190] [<ffffffff8113f6c9>] do_softirq+0x19/0x20 +[ 758.304442] [<ffffffff82f7caac>] netif_rx_ni+0xec/0x3a0 +[ 758.309892] [<ffffffff82753187>] tun_get_user+0xbe7/0x2410 +[ 758.326886] [<ffffffff82754bc5>] tun_chr_write_iter+0xd5/0x190 +[ 758.332918] [<ffffffff8151ce0d>] __vfs_write+0x30d/0x3f0 +[ 758.380578] [<ffffffff8151e9f1>] vfs_write+0x191/0x4e0 +[ 758.385913] [<ffffffff81520ff9>] SyS_write+0xd9/0x1c0 +[ 758.414182] [<ffffffff81006d96>] do_fast_syscall_32+0x326/0x8b0 +[ 758.420301] [<ffffffff838c406a>] sysenter_flags_fixed+0xd/0x17 diff --git a/pkg/report/testdata/linux/report/291 b/pkg/report/testdata/linux/report/291 new file mode 100644 index 000000000..47506e9d2 --- /dev/null +++ b/pkg/report/testdata/linux/report/291 @@ -0,0 +1,30 @@ +TITLE: INFO: rcu detected stall in rmdir + +[ 539.960575] INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 539.960585] (detected by 0, t=10502 jiffies, g=3970, c=3969, q=379) +[ 539.960592] All QSes seen, last rcu_preempt kthread activity 10502 (4294991162-4294980660), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 539.960608] syz-executor7 R running task 26968 2301 2298 0x00000000 +[ 539.960623] ffff8801db607c98 ffffffff813f5a7f ffffffff813f5886 0000000100003434 +[ 539.960633] 0000000100005d3a dffffc0000000000 ffff8801db621d40 ffffffff82ecd580 +[ 539.960643] ffff8801db607d68 ffffffff81401aa5 0000000000000000 ffff8801db607cd8 +[ 539.960645] Call Trace: +[ 539.960662] <IRQ> +[ 539.960663] [<ffffffff813f5a7f>] sched_show_task.cold.35+0x279/0x31f +[ 539.960679] [<ffffffff81401aa5>] rcu_check_callbacks.cold.69+0xc70/0xd27 +[ 539.960688] [<ffffffff81261680>] update_process_times+0x30/0x70 +[ 539.960697] [<ffffffff8128ec7a>] tick_sched_handle.isra.5+0x4a/0xf0 +[ 539.960704] [<ffffffff8128ed96>] tick_sched_timer+0x76/0x130 +[ 539.960720] [<ffffffff812643a7>] __hrtimer_run_queues+0x357/0xe30 +[ 539.960766] [<ffffffff81266891>] hrtimer_interrupt+0x1b1/0x430 +[ 539.960775] [<ffffffff8108f6c4>] local_apic_timer_interrupt+0x74/0xa0 +[ 539.960784] [<ffffffff8278ff1c>] smp_apic_timer_interrupt+0x7c/0xa0 +[ 539.960791] [<ffffffff8278d7ed>] apic_timer_interrupt+0x9d/0xb0 +[ 539.960802] <EOI> +[ 539.960812] [<ffffffff8278b454>] _raw_spin_lock_nested+0x44/0x50 +[ 539.960828] [<ffffffff81548219>] d_walk.part.10+0x1b9/0x710 +[ 539.960849] [<ffffffff815488f7>] shrink_dcache_parent+0xd7/0x130 +[ 539.960873] [<ffffffff81529bd9>] vfs_rmdir2+0x1c9/0x410 +[ 539.960880] [<ffffffff81533264>] do_rmdir+0x304/0x3b0 +[ 539.960916] [<ffffffff81534d9a>] SyS_rmdir+0x1a/0x20 +[ 539.960922] [<ffffffff8100554f>] do_syscall_64+0x19f/0x480 +[ 539.960930] [<ffffffff8278c053>] entry_SYSCALL_64_after_swapgs+0x5d/0xdb diff --git a/pkg/report/testdata/linux/report/292 b/pkg/report/testdata/linux/report/292 new file mode 100644 index 000000000..f8e401066 --- /dev/null +++ b/pkg/report/testdata/linux/report/292 @@ -0,0 +1,46 @@ +TITLE: INFO: rcu detected stall in wb_workfn + +[ 420.341960] INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 420.347923] (detected by 1, t=10502 jiffies, g=9244, c=9243, q=178) +[ 420.354558] All QSes seen, last rcu_preempt kthread activity 10502 (4294979208-4294968706), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 420.366779] kworker/u4:2 R running task 25560 32 2 0x00000008 +[ 420.374291] Workqueue: writeback wb_workfn (flush-8:0) +[ 420.379821] ffff8801db307ca0 ffffffff814249f9 0000000100000582 0000000100002e88 +[ 420.387871] dffffc0000000000 ffff8801db322880 ffffffff846e0300 ffff8801db307d70 +[ 420.395936] ffffffff8143035a 0000000000000000 ffffffff846def20 ffffffff846e0a18 +[ 420.403962] Call Trace: +[ 420.406526] <IRQ> [ 420.408595] [<ffffffff814249f9>] sched_show_task.cold.127+0x1c9/0x279 +[ 420.415289] [<ffffffff8143035a>] rcu_check_callbacks.cold.79+0xc70/0xd27 +[ 420.422204] [<ffffffff812a0ac0>] update_process_times+0x30/0x70 +[ 420.428339] [<ffffffff812cc695>] tick_sched_handle.isra.14+0x55/0xf0 +[ 420.434904] [<ffffffff812cceb2>] tick_sched_timer+0x72/0x120 +[ 420.446992] [<ffffffff812a4775>] __hrtimer_run_queues+0x375/0xe50 +[ 420.478026] [<ffffffff812a5cb1>] hrtimer_interrupt+0x1b1/0x430 +[ 420.484071] [<ffffffff810b2384>] local_apic_timer_interrupt+0x74/0xa0 +[ 420.490726] [<ffffffff83a0249c>] smp_apic_timer_interrupt+0x7c/0xa0 +[ 420.497210] [<ffffffff839fe630>] apic_timer_interrupt+0xa0/0xb0 +[ 420.503332] <EOI> [ 420.505397] [<ffffffff812da9eb>] ? smp_call_function_single+0x11b/0x360 +[ 420.518914] [<ffffffff812da9eb>] smp_call_function_single+0x11b/0x360 +[ 420.561741] [<ffffffff812db9ba>] smp_call_function_many+0x57a/0x6a0 +[ 420.579966] [<ffffffff810ed4a4>] native_flush_tlb_others+0xd4/0x510 +[ 420.597680] [<ffffffff810ed9ec>] flush_tlb_mm_range+0x10c/0x440 +[ 420.603832] [<ffffffff81500cdc>] ptep_clear_flush+0xec/0x110 +[ 420.609706] [<ffffffff814f05d0>] page_mkclean_one+0x100/0x220 +[ 420.622141] [<ffffffff814ee6f2>] rmap_walk_file+0x2c2/0x780 +[ 420.634853] [<ffffffff814f614d>] rmap_walk+0xed/0x180 +[ 420.640112] [<ffffffff814f6876>] page_mkclean+0x196/0x1d0 +[ 420.672156] [<ffffffff814586e0>] clear_page_dirty_for_io+0x200/0x4e0 +[ 420.678721] [<ffffffff8173c337>] mpage_submit_page+0x77/0x240 +[ 420.684675] [<ffffffff8173c8b2>] mpage_process_page_bufs+0x3b2/0x490 +[ 420.691270] [<ffffffff8173ff6f>] mpage_prepare_extent_to_map+0x52f/0x9a0 +[ 420.717326] [<ffffffff8174f609>] ext4_writepages+0xdb9/0x2e50 +[ 420.749218] [<ffffffff8145ef2f>] do_writepages+0xef/0x1d0 +[ 420.767364] [<ffffffff81601fa9>] __writeback_single_inode+0xd9/0x1020 +[ 420.774030] [<ffffffff81603a3c>] writeback_sb_inodes+0x4ac/0xe70 +[ 420.798721] [<ffffffff816044fb>] __writeback_inodes_wb+0xfb/0x1e0 +[ 420.805040] [<ffffffff81604c82>] wb_writeback+0x512/0xbd0 +[ 420.824129] [<ffffffff8160df6e>] wb_workfn+0x20e/0xdb0 +[ 420.841940] [<ffffffff8118d181>] process_one_work+0x7e1/0x1500 +[ 420.860701] [<ffffffff8118df76>] worker_thread+0xd6/0x10a0 +[ 420.872096] [<ffffffff8119d0ed>] kthread+0x26d/0x300 +[ 420.900433] [<ffffffff839fce9c>] ret_from_fork+0x5c/0x70 diff --git a/pkg/report/testdata/linux/report/293 b/pkg/report/testdata/linux/report/293 new file mode 100644 index 000000000..d08dcca52 --- /dev/null +++ b/pkg/report/testdata/linux/report/293 @@ -0,0 +1,50 @@ +TITLE: INFO: rcu detected stall in perf_mmap + +[ 323.391020] INFO: rcu_sched self-detected stall on CPU +[ 323.396363] 1-...: (64999 ticks this GP) idle=97a/140000000000001/0 softirq=7370/7370 fqs=16241 +[ 323.405345] (t=65000 jiffies g=4163 c=4162 q=64) +[ 323.410286] NMI backtrace for cpu 1 +[ 323.413913] CPU: 1 PID: 1885 Comm: syz-executor921 Not tainted 4.14.49+ #8 +[ 323.420908] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 323.430246] Call Trace: +[ 323.432801] <IRQ> +[ 323.434933] dump_stack+0x114/0x1cf +[ 323.457051] nmi_cpu_backtrace.cold.2+0x19/0x95 +[ 323.471164] nmi_trigger_cpumask_backtrace+0x102/0x143 +[ 323.476417] arch_trigger_cpumask_backtrace+0x19/0x20 +[ 323.481585] rcu_dump_cpu_stacks+0x177/0x1c4 +[ 323.485974] check_cpu_stall.isra.56.cold.74+0x289/0x4c3 +[ 323.569083] rcu_check_callbacks+0x2e4/0xe20 +[ 323.620799] update_process_times+0x35/0x70 +[ 323.625119] tick_sched_handle+0x8a/0x150 +[ 323.629243] tick_sched_timer+0x47/0x120 +[ 323.637661] __hrtimer_run_queues+0x4ea/0x1150 +[ 323.715212] hrtimer_interrupt+0x1b1/0x5b0 +[ 323.719432] smp_apic_timer_interrupt+0x1f9/0x870 +[ 323.756898] apic_timer_interrupt+0x8e/0xa0 +[ 323.761194] </IRQ> +[ 323.763409] RIP: 0010:lock_is_held_type+0x115/0x160 +[ 323.768396] RSP: 0018:ffff8801c5abf228 EFLAGS: 00000282 ORIG_RAX: ffffffffffffff10 +[ 323.776082] RAX: 0000000000000000 RBX: 1ffff10038b57e46 RCX: 0000000000000000 +[ 323.783326] RDX: 0000000000000000 RSI: ffffffff83efc6c0 RDI: 0000000000000246 +[ 323.790578] RBP: ffff8801c5abf2b0 R08: ffffffff81566839 R09: ffffffff8518b320 +[ 323.797822] R10: ffff8801c5abf700 R11: 0000000000000000 R12: 0000000000000000 +[ 323.805074] R13: ffffffff8385a220 R14: 00000000000002eb R15: 0000000000000000 +[ 323.832368] __might_sleep+0x9a/0x190 +[ 323.848601] __mutex_lock+0x143/0x19c0 +[ 323.917528] mutex_lock_nested+0x1b/0x20 +[ 323.925769] perf_mmap+0x689/0x17f0 +[ 323.969686] mmap_region+0xba0/0x1660 +[ 323.998425] do_mmap+0x730/0xea0 +[ 324.018118] vm_mmap_pgoff+0x1e3/0x270 +[ 324.033674] SyS_mmap_pgoff+0x46d/0x630 +[ 324.055175] SyS_mmap+0x1b/0x30 +[ 324.058439] do_syscall_64+0x251/0x750 +[ 324.087114] entry_SYSCALL_64_after_hwframe+0x42/0xb7 +[ 324.092288] RIP: 0033:0x445e09 +[ 324.095451] RSP: 002b:00007eff4d398d98 EFLAGS: 00000212 ORIG_RAX: 0000000000000009 +[ 324.103133] RAX: ffffffffffffffda RBX: 00000000006dac5c RCX: 0000000000445e09 +[ 324.110376] RDX: 0000000000000000 RSI: 0000000000001000 RDI: 00000000201bf000 +[ 324.117619] RBP: 00000000006dac58 R08: 0000000000000301 R09: 0000000000000000 +[ 324.124872] R10: 0080000000004011 R11: 0000000000000212 R12: 7479625f6e695f65 +[ 324.132116] R13: 676173752e424d32 R14: 2e626c7465677568 R15: 0000000000000009 diff --git a/pkg/report/testdata/linux/report/294 b/pkg/report/testdata/linux/report/294 new file mode 100644 index 000000000..bf34a6e0c --- /dev/null +++ b/pkg/report/testdata/linux/report/294 @@ -0,0 +1,44 @@ +TITLE: INFO: rcu detected stall in lo_compat_ioctl + +[ 480.801028] INFO: rcu_sched self-detected stall on CPU +[ 480.806333] 0-...: (64962 ticks this GP) idle=ff6/140000000000001/0 softirq=6610/6610 fqs=16220 +[ 480.815311] (t=65000 jiffies g=3400 c=3399 q=180) +[ 480.820444] NMI backtrace for cpu 0 +[ 480.824066] CPU: 0 PID: 1932 Comm: syz-executor0 Not tainted 4.14.35+ #15 +[ 480.830976] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 480.840409] Call Trace: +[ 480.842967] <IRQ> +[ 480.845148] dump_stack+0x114/0x1cf +[ 480.867367] nmi_cpu_backtrace.cold.2+0x19/0x95 +[ 480.881500] nmi_trigger_cpumask_backtrace+0x102/0x143 +[ 480.886763] arch_trigger_cpumask_backtrace+0x19/0x20 +[ 480.891968] rcu_dump_cpu_stacks+0x177/0x1c4 +[ 480.896355] check_cpu_stall.isra.56.cold.75+0x289/0x4b7 +[ 480.975100] rcu_check_callbacks+0x2e4/0xe20 +[ 481.026890] update_process_times+0x35/0x70 +[ 481.031242] tick_sched_handle+0x8a/0x150 +[ 481.035364] tick_sched_timer+0x47/0x120 +[ 481.043793] __hrtimer_run_queues+0x4ea/0x1150 +[ 481.122814] hrtimer_interrupt+0x1b1/0x5b0 +[ 481.127041] smp_apic_timer_interrupt+0x1f9/0x870 +[ 481.169084] apic_timer_interrupt+0x8e/0xa0 +[ 481.173377] </IRQ> +[ 481.175588] RIP: 0010:lo_ioctl+0x15e4/0x1b50 +[ 481.179969] RSP: 0018:ffff8801cd6efb78 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff10 +[ 481.187652] RAX: 1ffff100395975ef RBX: ffff8801cacbadc0 RCX: dffffc0000000000 +[ 481.194907] RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff8801cacbaf78 +[ 481.202165] RBP: ffff8801cd6efbf0 R08: 0000000000000001 R09: 0000000000000000 +[ 481.209417] R10: 0000000000000003 R11: 0000000000000001 R12: ffff8801caf99c40 +[ 481.216664] R13: ffff8801db016320 R14: ffff8801d51e4b48 R15: ffff8801db016240 +[ 481.236297] lo_compat_ioctl+0xb3/0x140 +[ 481.244057] compat_blkdev_ioctl+0x39c/0x1950 +[ 481.261895] compat_SyS_ioctl+0x19d/0x3030 +[ 481.281601] do_fast_syscall_32+0x3c1/0xbf1 +[ 481.317738] entry_SYSENTER_compat+0x84/0x96 +[ 481.322121] RIP: 0023:0xf7f71c99 +[ 481.325470] RSP: 002b:00000000fff4d7ec EFLAGS: 00000286 ORIG_RAX: 0000000000000036 +[ 481.333156] RAX: ffffffffffffffda RBX: 0000000000000004 RCX: 0000000000004c00 +[ 481.340410] RDX: 0000000000000003 RSI: 0000000000000000 RDI: 0000000000000000 +[ 481.347665] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000 +[ 481.354909] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000 +[ 481.362160] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/295 b/pkg/report/testdata/linux/report/295 new file mode 100644 index 000000000..2c903954c --- /dev/null +++ b/pkg/report/testdata/linux/report/295 @@ -0,0 +1,58 @@ +TITLE: INFO: rcu detected stall in netlink_sendmsg + +[ 355.572806] INFO: rcu_sched self-detected stall on CPU +[ 355.578254] 0: (104999 ticks this GP) idle=ab9/140000000000001/0 softirq=76582/76582 fqs=34894 +[ 355.587151] (t=105000 jiffies g=36307 c=36306 q=2303) +[ 355.592663] Task dump for CPU 0: +[ 355.596007] syz-executor7 R running task on cpu 0 0 19712 4019 0x0000000c 250568154159 +[ 355.605206] 0000000000000fb3 00000000ccf50a4a ffff8801daa07bd8 ffffffff816f0dc7 +[ 355.613296] ffff8801daa1f7c0 0000000000000000 dffffc0000000000 ffffffff83fca380 +[ 355.621298] ffffffff83fca3f4 ffff8801daa07be8 ffffffff816f154c ffff8801daa07c08 +[ 355.629309] Call Trace: +[ 355.631878] <IRQ> [<ffffffff816f0dc7>] _sched_show_task+0x31a/0x325 +[ 355.638595] [<ffffffff816f154c>] sched_show_task+0x10/0x12 +[ 355.644287] [<ffffffff816f16f1>] dump_cpu_task+0x7e/0x83 +[ 355.649807] [<ffffffff816faea8>] rcu_dump_cpu_stacks+0x155/0x169 +[ 355.656012] [<ffffffff816fb851>] rcu_check_callbacks.cold.76+0x61e/0xc77 +[ 355.668696] [<ffffffff814f6faf>] update_process_times+0x3f/0x70 +[ 355.674817] [<ffffffff815204ba>] tick_sched_handle.isra.16+0x5a/0x100 +[ 355.681474] [<ffffffff81520b6a>] tick_sched_timer+0x7a/0x130 +[ 355.687337] [<ffffffff814fa0e5>] __hrtimer_run_queues+0x3a5/0xc50 +[ 355.725548] [<ffffffff814fb30e>] hrtimer_interrupt+0x18e/0x400 +[ 355.731583] [<ffffffff8129b744>] local_apic_timer_interrupt+0x74/0x90 +[ 355.738224] [<ffffffff81012dff>] smp_apic_timer_interrupt+0xdf/0x130 +[ 355.744785] [<ffffffff81011c72>] smp_apic_timer_interrupt_entry_after_kaiser_bti+0x20/0x2e +[ 355.753246] <EOI> [<ffffffff81494f40>] ? check_noncircular+0x20/0x20 +[ 355.766683] [<ffffffff815e3567>] is_ftrace_trampoline+0x37/0x120 +[ 355.772905] [<ffffffff813a1928>] __kernel_text_address+0x88/0xc0 +[ 355.779116] [<ffffffff81245324>] print_context_stack+0x54/0xe0 +[ 355.785183] [<ffffffff8124441d>] dump_trace+0x16d/0x320 +[ 355.790612] [<ffffffff81265a0b>] save_stack_trace+0x2b/0x50 +[ 355.796389] [<ffffffff8183fd43>] save_stack+0x43/0xd0 +[ 356.049567] [<ffffffff8183ffd4>] kasan_kmalloc+0xc4/0xe0 +[ 356.055079] [<ffffffff81840582>] kasan_slab_alloc+0x12/0x20 +[ 356.060852] [<ffffffff8183a96d>] kmem_cache_alloc+0x14d/0x710 +[ 356.078645] [<ffffffff81df032e>] ida_pre_get+0x12e/0x2b0 +[ 356.096113] [<ffffffff81df0589>] ida_simple_get+0xd9/0x1e0 +[ 356.117746] [<ffffffff81a32b67>] __kernfs_new_node+0x97/0x2c0 +[ 356.123707] [<ffffffff81a36535>] kernfs_new_node+0x85/0xf0 +[ 356.129409] [<ffffffff81a36ec2>] kernfs_create_dir_ns+0x42/0x140 +[ 356.135620] [<ffffffff81a3dd23>] sysfs_create_dir_ns+0xc3/0x1d0 +[ 356.141741] [<ffffffff81df2be4>] kobject_add_internal+0x284/0x990 +[ 356.148036] [<ffffffff81df33d7>] kobject_init_and_add+0xe7/0x140 +[ 356.160732] [<ffffffff82af29a4>] netdev_queue_update_kobjects+0xf4/0x2b0 +[ 356.167638] [<ffffffff82af2eaf>] netdev_register_kobject+0x22f/0x2f0 +[ 356.174196] [<ffffffff82a96121>] register_netdevice+0x7d1/0xf60 +[ 356.205338] [<ffffffff82e6da68>] ip_tunnel_newlink+0x378/0x770 +[ 356.217510] [<ffffffff82e83a52>] ipgre_newlink+0x102/0x160 +[ 356.247884] [<ffffffff82acf8f5>] rtnl_newlink+0x1055/0x15c0 +[ 356.316833] [<ffffffff82ad0314>] rtnetlink_rcv_msg+0x4b4/0x700 +[ 356.353912] [<ffffffff82c13dc3>] netlink_rcv_skb+0x143/0x370 +[ 356.365750] [<ffffffff82ac074f>] rtnetlink_rcv+0x2f/0x40 +[ 356.371288] [<ffffffff82c128c7>] netlink_unicast+0x517/0x740 +[ 356.395498] [<ffffffff82c132d4>] netlink_sendmsg+0x7e4/0xc90 +[ 356.420047] [<ffffffff82a088a8>] sock_sendmsg+0xe8/0x150 +[ 356.425579] [<ffffffff82a0c3f0>] ___sys_sendmsg+0x760/0x8b0 +[ 356.475821] [<ffffffff82a0f0eb>] __sys_sendmsg+0xdb/0x190 +[ 356.504375] [<ffffffff82a0f1d2>] SyS_sendmsg+0x32/0x50 +[ 356.509728] [<ffffffff8307ddd3>] entry_SYSCALL_64_fastpath+0x12/0x17 diff --git a/pkg/report/testdata/linux/report/296 b/pkg/report/testdata/linux/report/296 new file mode 100644 index 000000000..c5660be7a --- /dev/null +++ b/pkg/report/testdata/linux/report/296 @@ -0,0 +1,36 @@ +TITLE: BUG: soft lockup in mount + +[ 848.443720] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 123s! [syz-executor694:3996] +[ 848.452125] Modules linked in: +[ 848.455412] irq event stamp: 933704 +[ 848.459142] hardirqs last enabled at (933703): [<ffffffff839f9669>] restore_regs_and_iret+0x0/0x1d +[ 848.468415] hardirqs last disabled at (933704): [<ffffffff839fa62b>] apic_timer_interrupt+0x9b/0xb0 +[ 848.477698] softirqs last enabled at (921172): [<ffffffff839ffb1c>] __do_softirq+0x46c/0x937 +[ 848.486386] softirqs last disabled at (921163): [<ffffffff8114b237>] irq_exit+0x147/0x190 +[ 848.496037] CPU: 1 PID: 3996 Comm: syz-executor694 Not tainted 4.9.111-g03c70fe #10 +[ 848.503812] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 848.513416] task: ffff8801d8ac3000 task.stack: ffff8801b70a0000 +[ 848.519454] RIP: 0010:[<ffffffff8160fc09>] [<ffffffff8160fc09>] change_mnt_propagation+0xa9/0xbd0 +[ 848.528820] RSP: 0018:ffff8801b70a7af0 EFLAGS: 00000246 +[ 848.534258] RAX: ffff8801d8ac3000 RBX: ffff8801cda6e400 RCX: ffff8801cda6e468 +[ 848.541526] RDX: 1ffff1003723a2c4 RSI: ffffffff8160fbe8 RDI: ffff8801b91d1620 +[ 848.548806] RBP: ffff8801b70a7b50 R08: ffff8801cda6e468 R09: 0000000000000001 +[ 848.556086] R10: 0000000000000000 R11: 1ffff1003b15872c R12: dffffc0000000000 +[ 848.563348] R13: ffff8801b91d1600 R14: ffff8801d9c47340 R15: ffffed0039b4dc84 +[ 848.570605] FS: 0000000000000000(0000) GS:ffff8801db300000(0063) knlGS:00000000f7667b40 +[ 848.578828] CS: 0010 DS: 002b ES: 002b CR0: 0000000080050033 +[ 848.584692] CR2: 00000000f76cadb0 CR3: 00000001ca4fa000 CR4: 00000000001606f0 +[ 848.592068] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 848.599318] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 848.606567] Stack: +[ 848.608707] ffff8801b70a7b20 ffffffff815db8af 0000000000000000 ffff8801cda6e4f0 +[ 848.616764] ffff8801cda6e420 ffff8801cda6e4e0 00040000b70a7b50 ffff8801cda6e490 +[ 848.624800] dffffc0000000000 ffff8801cda6e400 ffff8801d9c469a0 ffff8801b70a7bf0 +[ 848.632818] Call Trace: +[ 848.641493] [<ffffffff815dbfd6>] umount_tree+0x326/0x820 +[ 848.652638] [<ffffffff815e41e0>] attach_recursive_mnt+0x750/0x8f0 +[ 848.675872] [<ffffffff815e44ef>] graft_tree+0x16f/0x1f0 +[ 848.681308] [<ffffffff815e6eaf>] do_mount+0x1f5f/0x2740 +[ 848.724324] [<ffffffff8168c3ec>] compat_SyS_mount+0x4fc/0xff0 +[ 848.743149] [<ffffffff81006da7>] do_fast_syscall_32+0x2f7/0x870 +[ 848.755940] [<ffffffff839fa3d0>] entry_SYSENTER_compat+0x90/0xa2 diff --git a/pkg/report/testdata/linux/report/297 b/pkg/report/testdata/linux/report/297 new file mode 100644 index 000000000..586be8c13 --- /dev/null +++ b/pkg/report/testdata/linux/report/297 @@ -0,0 +1,39 @@ +TITLE: BUG: soft lockup in blkdev_close + +[ 601.962573] watchdog: BUG: soft lockup - CPU#1 stuck for 123s! [syz-executor1:24793] +[ 601.970515] Modules linked in: +[ 601.973739] CPU: 1 PID: 24793 Comm: syz-executor1 Not tainted 4.18.0-rc4+ #24 +[ 601.981015] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 601.990409] RIP: 0010:msan_get_shadow_origin_ptr+0x6f/0x300 +[ 601.996116] Code: 8b 48 0f 45 d8 80 3c 25 00 f0 a0 8b 00 0f 84 08 02 00 00 65 4c 8b 2c 25 80 fc 02 00 41 83 bd f4 08 00 00 00 0f 85 f1 01 00 00 <49> 81 fc 01 10 00 00 72 23 0f b6 ca 45 31 f6 48 c7 c7 ba 47 d7 8a +[ 602.016312] RSP: 0018:ffff8800b3a2f208 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 602.024043] RAX: ffffffff8ba13000 RBX: ffffffff8ba11000 RCX: ffff880103d43b00 +[ 602.031323] RDX: 0000000000000000 RSI: 0000000000000004 RDI: ffff8800b3a2f2d8 +[ 602.038604] RBP: ffff8800b3a2f260 R08: 0000000000000000 R09: 0000000000000002 +[ 602.045887] R10: 0000000000000000 R11: ffffffff8117d090 R12: 0000000000000004 +[ 602.053171] R13: ffff880103d43b00 R14: ffffffff8ba10000 R15: ffff8800b3a2f2d8 +[ 602.060458] FS: 00007f4dc65dc700(0000) GS:ffff88021fd00000(0000) knlGS:0000000000000000 +[ 602.068697] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 602.074586] CR2: 00000000004544da CR3: 00000000b3c12000 CR4: 00000000001406e0 +[ 602.081868] DR0: 0000000020000000 DR1: 0000000020000000 DR2: 0000000000000000 +[ 602.089149] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000600 +[ 602.096425] Call Trace: +[ 602.104092] __msan_metadata_ptr_for_load_4+0x10/0x20 +[ 602.109301] smp_call_function_single+0x363/0x500 +[ 602.122402] smp_call_function_many+0x672/0xe30 +[ 602.135338] on_each_cpu_cond+0x335/0x420 +[ 602.148630] invalidate_bh_lrus+0x54/0x60 +[ 602.152805] __blkdev_put+0x44b/0xcc0 +[ 602.160518] blkdev_put+0x435/0x570 +[ 602.164201] blkdev_close+0xa6/0xf0 +[ 602.167872] __fput+0x458/0xa30 +[ 602.174520] ____fput+0x37/0x40 +[ 602.177819] task_work_run+0x22e/0x2b0 +[ 602.181766] do_exit+0x110e/0x3930 +[ 602.185376] do_group_exit+0x1a0/0x360 +[ 602.189287] get_signal+0x15c3/0x2190 +[ 602.216814] do_signal+0xca/0x2060 +[ 602.229420] prepare_exit_to_usermode+0x297/0x430 +[ 602.234312] syscall_return_slowpath+0x108/0x800 +[ 602.243411] do_syscall_64+0x1ad/0x230 +[ 602.247324] entry_SYSCALL_64_after_hwframe+0x63/0xe7 diff --git a/pkg/report/testdata/linux/report/298 b/pkg/report/testdata/linux/report/298 new file mode 100644 index 000000000..1ecf5e7dc --- /dev/null +++ b/pkg/report/testdata/linux/report/298 @@ -0,0 +1,58 @@ +TITLE: INFO: rcu detected stall in netlink_sendmsg + +[ 287.048281] INFO: rcu_sched self-detected stall on CPU +[ 287.053694] 0: (104999 ticks this GP) idle=769/140000000000001/0 softirq=28286/28286 fqs=34911 +[ 287.062584] (t=105000 jiffies g=13648 c=13647 q=595) +[ 287.067988] Task dump for CPU 0: +[ 287.071369] syz-executor2 R running task on cpu 0 0 7831 4518 0x2002000c 181735472838 +[ 287.080530] 00000000000011a6 000000001a7b0e17 ffff8801daa07bd8 ffffffff816ecb37 +[ 287.088617] ffff8801daa1f780 0000000000000000 dffffc0000000000 ffffffff83fca0c0 +[ 287.096597] ffffffff83fca134 ffff8801daa07be8 ffffffff816ed2bc ffff8801daa07c08 +[ 287.104578] Call Trace: +[ 287.107132] <IRQ> [<ffffffff816ecb37>] _sched_show_task+0x31a/0x325 +[ 287.113816] [<ffffffff816ed2bc>] sched_show_task+0x10/0x12 +[ 287.119500] [<ffffffff816ed461>] dump_cpu_task+0x7e/0x83 +[ 287.125012] [<ffffffff816f6c18>] rcu_dump_cpu_stacks+0x155/0x169 +[ 287.131219] [<ffffffff816f75c1>] rcu_check_callbacks.cold.76+0x61e/0xc77 +[ 287.143892] [<ffffffff814f2c8f>] update_process_times+0x3f/0x70 +[ 287.150010] [<ffffffff8151c19a>] tick_sched_handle.isra.16+0x5a/0x100 +[ 287.156649] [<ffffffff8151c84a>] tick_sched_timer+0x7a/0x130 +[ 287.162510] [<ffffffff814f5dc5>] __hrtimer_run_queues+0x3a5/0xc50 +[ 287.200662] [<ffffffff814f6fee>] hrtimer_interrupt+0x18e/0x400 +[ 287.206695] [<ffffffff81297744>] local_apic_timer_interrupt+0x74/0x90 +[ 287.213352] [<ffffffff81012dff>] smp_apic_timer_interrupt+0xdf/0x130 +[ 287.219929] [<ffffffff81011c72>] smp_apic_timer_interrupt_entry_after_kaiser_bti+0x20/0x2e +[ 287.228401] <EOI> [<ffffffff82a7a1ab>] ? alloc_netdev_mqs+0x97b/0xc50 +[ 287.241566] [<ffffffff8139d8d8>] __kernel_text_address+0x88/0xc0 +[ 287.247777] [<ffffffff81241324>] print_context_stack+0x54/0xe0 +[ 287.253814] [<ffffffff8124041d>] dump_trace+0x16d/0x320 +[ 287.259241] [<ffffffff81261a0b>] save_stack_trace+0x2b/0x50 +[ 287.265014] [<ffffffff8183c0f3>] save_stack+0x43/0xd0 +[ 287.412837] [<ffffffff8183c384>] kasan_kmalloc+0xc4/0xe0 +[ 287.418346] [<ffffffff8183c932>] kasan_slab_alloc+0x12/0x20 +[ 287.424118] [<ffffffff81835be3>] kmem_cache_alloc+0x5e3/0x740 +[ 287.436125] [<ffffffff81dedefe>] ida_pre_get+0x12e/0x2b0 +[ 287.453551] [<ffffffff81dee159>] ida_simple_get+0xd9/0x1e0 +[ 287.471649] [<ffffffff81a2ee37>] __kernfs_new_node+0x97/0x2c0 +[ 287.477598] [<ffffffff81a32805>] kernfs_new_node+0x85/0xf0 +[ 287.483282] [<ffffffff81a37510>] __kernfs_create_file+0x50/0x2a0 +[ 287.489487] [<ffffffff81a39526>] sysfs_add_file_mode_ns+0x1f6/0x510 +[ 287.495955] [<ffffffff81a398d0>] sysfs_create_file_ns+0x90/0xb0 +[ 287.502085] [<ffffffff81df09a6>] kobject_add_internal+0x476/0x990 +[ 287.508396] [<ffffffff81df0fa7>] kobject_init_and_add+0xe7/0x140 +[ 287.521073] [<ffffffff82af31d4>] netdev_queue_update_kobjects+0xf4/0x2b0 +[ 287.527973] [<ffffffff82af36df>] netdev_register_kobject+0x22f/0x2f0 +[ 287.534545] [<ffffffff82a96b01>] register_netdevice+0x7d1/0xf60 +[ 287.565577] [<ffffffff82e69108>] ip_tunnel_newlink+0x378/0x770 +[ 287.577761] [<ffffffff82e7f0f2>] ipgre_newlink+0x102/0x160 +[ 287.608141] [<ffffffff82ad02d5>] rtnl_newlink+0x1055/0x15c0 +[ 287.699732] [<ffffffff82ad0cf4>] rtnetlink_rcv_msg+0x4b4/0x700 +[ 287.736738] [<ffffffff82c10ad3>] netlink_rcv_skb+0x143/0x370 +[ 287.748552] [<ffffffff82ac112f>] rtnetlink_rcv+0x2f/0x40 +[ 287.754063] [<ffffffff82c0f5d7>] netlink_unicast+0x517/0x740 +[ 287.778219] [<ffffffff82c0ffe4>] netlink_sendmsg+0x7e4/0xc90 +[ 287.802726] [<ffffffff82a09048>] sock_sendmsg+0xe8/0x150 +[ 287.808240] [<ffffffff82a0cb90>] ___sys_sendmsg+0x760/0x8b0 +[ 287.858329] [<ffffffff82a0f88b>] __sys_sendmsg+0xdb/0x190 +[ 287.888147] [<ffffffff82b2c18f>] compat_SyS_sendmsg+0x2f/0x40 +[ 287.894111] [<ffffffff8307b2f0>] sysenter_dispatch+0xf/0x32 diff --git a/pkg/report/testdata/linux/report/299 b/pkg/report/testdata/linux/report/299 new file mode 100644 index 000000000..9da269095 --- /dev/null +++ b/pkg/report/testdata/linux/report/299 @@ -0,0 +1,24 @@ +TITLE: INFO: rcu detected stall in sched_ttwu_pending + +[ 575.758025] INFO: rcu_sched detected stalls on CPUs/tasks: +[ 575.764179] (detected by 1, t=300086 jiffies, g=27161, c=27160, q=118) +[ 575.771051] All QSes seen, last rcu_sched kthread activity 300086 (4295238516-4294938430), jiffies_till_next_fqs=3, root ->qsmask 0x0 +[ 575.783183] swapper/1 R running task on cpu 1 0 0 1 0x00000008 0 +[ 575.791404] 0000000000000001 18186e061cc9dc83 ffff8801dab07c50 ffffffff816ecb37 +[ 575.799432] ffff8801dab20880 0000000100042374 00000000ffff8f3e ffffffff83fca0c0 +[ 575.807415] 0000000000000003 ffff8801dab07c60 ffffffff816ed2bc ffff8801dab07d30 +[ 575.815434] Call Trace: +[ 575.817998] <IRQ> [<ffffffff816ecb37>] _sched_show_task+0x31a/0x325 +[ 575.824729] [<ffffffff816ed2bc>] sched_show_task+0x10/0x12 +[ 575.830434] [<ffffffff816f7b9d>] rcu_check_callbacks.cold.76+0xbfa/0xc77 +[ 575.837357] [<ffffffff814f2c8f>] update_process_times+0x3f/0x70 +[ 575.843494] [<ffffffff8151c19a>] tick_sched_handle.isra.16+0x5a/0x100 +[ 575.850145] [<ffffffff8151c84a>] tick_sched_timer+0x7a/0x130 +[ 575.856014] [<ffffffff814f5dc5>] __hrtimer_run_queues+0x3a5/0xc50 +[ 575.894248] [<ffffffff814f6fee>] hrtimer_interrupt+0x18e/0x400 +[ 575.900294] [<ffffffff81297744>] local_apic_timer_interrupt+0x74/0x90 +[ 575.906949] [<ffffffff81012dff>] smp_apic_timer_interrupt+0xdf/0x130 +[ 575.913516] [<ffffffff81011c72>] smp_apic_timer_interrupt_entry_after_kaiser_bti+0x20/0x2e +[ 575.929641] [<ffffffff813dc8da>] sched_ttwu_pending+0x10a/0x200 +[ 575.941724] [<ffffffff8144bdbe>] cpu_startup_entry+0x14e/0x990 +[ 575.974371] [<ffffffff812946d5>] start_secondary+0x305/0x3e0 diff --git a/pkg/report/testdata/linux/report/300 b/pkg/report/testdata/linux/report/300 new file mode 100644 index 000000000..8602edef5 --- /dev/null +++ b/pkg/report/testdata/linux/report/300 @@ -0,0 +1,44 @@ +TITLE: INFO: rcu detected stall in uhaul_release + +[ 3481.239960] INFO: rcu_sched detected stalls on CPUs/tasks: +[ 3481.239969] (detected by 1, t=105002 jiffies, g=687946, c=687945, q=520) +[ 3481.239975] All QSes seen, last rcu_sched kthread activity 103964 (4298143477-4298039513), jiffies_till_next_fqs=3, root ->qsmask 0x0 +[ 3481.239987] syz-executor0 R running task on cpu 1 0 315571 315308 0x0000000e 3375506638533 +[ 3481.240033] 000000000004cfac 000000007b396287 ffff8801dab07c50 ffffffff816efb87 +[ 3481.240042] ffff8801dab20880 00000001003076f5 00000001002ee0d9 ffffffff83fca0c0 +[ 3481.240050] 0000000000000003 ffff8801dab07c60 ffffffff816f030c ffff8801dab07d30 +[ 3481.240052] Call Trace: +[ 3481.240067] <IRQ> [<ffffffff816efb87>] _sched_show_task+0x31a/0x325 +[ 3481.240075] [<ffffffff816f030c>] sched_show_task+0x10/0x12 +[ 3481.240082] [<ffffffff816fabed>] rcu_check_callbacks.cold.76+0xbfa/0xc77 +[ 3481.240092] [<ffffffff814f5cdf>] update_process_times+0x3f/0x70 +[ 3481.240101] [<ffffffff8151f1ea>] tick_sched_handle.isra.16+0x5a/0x100 +[ 3481.240108] [<ffffffff8151f89a>] tick_sched_timer+0x7a/0x130 +[ 3481.240115] [<ffffffff814f8e15>] __hrtimer_run_queues+0x3a5/0xc50 +[ 3481.240160] [<ffffffff814fa03e>] hrtimer_interrupt+0x18e/0x400 +[ 3481.240169] [<ffffffff8129a744>] local_apic_timer_interrupt+0x74/0x90 +[ 3481.240177] [<ffffffff81012dff>] smp_apic_timer_interrupt+0xdf/0x130 +[ 3481.240186] [<ffffffff81011c72>] smp_apic_timer_interrupt_entry_after_kaiser_bti+0x20/0x2e +[ 3481.240196] <EOI> [<ffffffff815bcdb1>] ? __sanitizer_cov_trace_pc+0x21/0x60 +[ 3481.240204] [<ffffffff8152dedc>] smp_call_function_single+0x26c/0x330 +[ 3481.240241] [<ffffffff8152eaa5>] smp_cfm_core+0x6a5/0x7e0 +[ 3481.240269] [<ffffffff8152ee6b>] on_each_cpu+0x6b/0x120 +[ 3481.240278] [<ffffffff812dd836>] flush_tlb_kernel_range+0x86/0xe0 +[ 3481.240294] [<ffffffff817ea3cc>] __purge_vmap_area_lazy+0x3ec/0xf40 +[ 3481.240333] [<ffffffff817eb5df>] free_vmap_area_noflush+0x1bf/0x200 +[ 3481.240362] [<ffffffff817ee77e>] remove_vm_area+0x13e/0x1c0 +[ 3481.240368] [<ffffffff817ee849>] __vunmap+0x49/0x330 +[ 3481.240374] [<ffffffff817eebf5>] vfree+0x55/0xe0 +[ 3481.240381] [<ffffffff8177bc6b>] kvfree+0x3b/0x60 +[ 3481.240389] [<ffffffff82430b98>] uhaul_rx_release+0x268/0x5a0 +[ 3481.240412] [<ffffffff82428bee>] uhaul_release+0xce/0x3a0 +[ 3481.240432] [<ffffffff82a06882>] sock_release+0x92/0x1b0 +[ 3481.240438] [<ffffffff82a069bb>] sock_close+0x1b/0x20 +[ 3481.240446] [<ffffffff8189fbae>] __fput+0x22e/0x6d0 +[ 3481.240453] [<ffffffff818a00da>] ____fput+0x1a/0x20 +[ 3481.240462] [<ffffffff813a0710>] task_work_run+0x1a0/0x240 +[ 3481.240470] [<ffffffff8136f8e8>] get_signal+0x1098/0x14a0 +[ 3481.240486] [<ffffffff8123b14e>] do_signal+0x7e/0x400 +[ 3481.240524] [<ffffffff8101a530>] prepare_exit_to_usermode+0x170/0x390 +[ 3481.240531] [<ffffffff8101a817>] syscall_return_slowpath+0xc7/0x5c0 +[ 3481.240538] [<ffffffff8307e1e3>] int_ret_from_sys_call+0x25/0xba diff --git a/pkg/report/testdata/linux/report/65 b/pkg/report/testdata/linux/report/65 index d6cc73b35..e8b0d8381 100644 --- a/pkg/report/testdata/linux/report/65 +++ b/pkg/report/testdata/linux/report/65 @@ -1,4 +1,4 @@ -TITLE: BUG: soft lockup in smp_call_function_many +TITLE: BUG: soft lockup in jump_label_update_timeout [ 247.938942] watchdog: BUG: soft lockup - CPU#0 stuck for 134s! [kworker/0:2:1400] [ 247.946595] Modules linked in: @@ -22,86 +22,18 @@ TITLE: BUG: soft lockup in smp_call_function_many [ 248.078408] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 248.084262] CR2: 0000000020335ffc CR3: 00000001d0869000 CR4: 00000000001426f0 [ 248.091506] Call Trace: -[ 248.094075] ? add_nops+0x90/0x90 -[ 248.097504] ? generic_exec_single+0x5b0/0x5b0 -[ 248.102062] ? __mutex_lock+0x16f/0x1a80 -[ 248.106093] ? arch_jump_label_transform+0x1b/0x40 -[ 248.110995] ? print_usage_bug+0x3f0/0x3f0 -[ 248.115213] ? _find_next_bit+0xee/0x120 -[ 248.119250] ? add_nops+0x90/0x90 -[ 248.122676] ? find_next_bit+0x27/0x30 [ 248.126547] smp_call_function_many+0x773/0x930 -[ 248.131192] ? smp_call_function_many+0x773/0x930 -[ 248.136013] ? add_nops+0x90/0x90 -[ 248.139449] ? smp_call_function_single_async+0x2d0/0x2d0 -[ 248.144960] ? mark_held_locks+0xb2/0x100 -[ 248.149082] ? text_poke+0x336/0x530 -[ 248.152772] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 248.157763] ? apic_has_pending_timer+0x80/0x2a0 -[ 248.162488] ? trace_hardirqs_on+0xd/0x10 -[ 248.166606] ? add_nops+0x90/0x90 -[ 248.170033] ? apic_has_pending_timer+0x80/0x2a0 [ 248.174760] on_each_cpu+0x3d/0x1b0 -[ 248.178359] ? apic_has_pending_timer+0x7f/0x2a0 [ 248.183088] text_poke_bp+0xe4/0x170 -[ 248.186774] ? __mutex_unlock_slowpath+0xe9/0xac0 -[ 248.191590] ? poke_int3_handler+0x90/0x90 -[ 248.195800] ? wait_for_completion+0x7b0/0x7b0 -[ 248.200361] ? apic_has_pending_timer+0x7f/0x2a0 [ 248.205094] __jump_label_transform.isra.0+0x6a5/0x8a0 -[ 248.210352] ? bug_at+0x20/0x20 -[ 248.213606] ? check_noncircular+0x20/0x20 -[ 248.217814] ? perf_trace_lock_acquire+0xe3/0x980 -[ 248.222633] ? modules_open+0xa0/0xa0 [ 248.226421] arch_jump_label_transform+0x2f/0x40 [ 248.231152] __jump_label_update+0x207/0x2d0 -[ 248.235539] ? static_key_count+0x110/0x110 -[ 248.239838] ? atomic_dec_and_mutex_lock+0x112/0x150 -[ 248.244917] ? mutex_spin_on_owner+0xb50/0xb50 [ 248.249480] jump_label_update+0x22c/0x2b0 [ 248.253695] static_key_slow_dec_cpuslocked+0x176/0x1d0 -[ 248.259034] ? static_key_slow_inc+0x3c0/0x3c0 -[ 248.263596] ? rcu_read_lock_sched_held+0x108/0x120 [ 248.268594] jump_label_update_timeout+0x1f/0x30 [ 248.273326] process_one_work+0xbfd/0x1be0 -[ 248.277556] ? pwq_dec_nr_in_flight+0x450/0x450 -[ 248.282197] ? finish_task_switch+0x1d3/0x740 -[ 248.286664] ? finish_task_switch+0x1aa/0x740 -[ 248.291159] ? perf_trace_lock_acquire+0xe3/0x980 -[ 248.295987] ? perf_trace_lock+0x900/0x900 -[ 248.300191] ? __sched_text_start+0x8/0x8 -[ 248.304316] ? lock_downgrade+0x980/0x980 -[ 248.308440] ? __wake_up_common_lock+0x190/0x310 -[ 248.313189] ? find_held_lock+0x39/0x1d0 -[ 248.317242] ? lock_acquire+0x1d5/0x580 -[ 248.321189] ? worker_thread+0x4a3/0x1990 -[ 248.325321] ? lock_release+0xda0/0xda0 -[ 248.329271] ? worker_pool_assign_id+0x1b0/0x1b0 -[ 248.334004] ? do_raw_spin_trylock+0x190/0x190 [ 248.338580] worker_thread+0x223/0x1990 -[ 248.342560] ? process_one_work+0x1be0/0x1be0 -[ 248.347034] ? _raw_spin_unlock_irq+0x27/0x70 -[ 248.351505] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 248.356494] ? trace_hardirqs_on+0xd/0x10 -[ 248.360617] ? finish_task_switch+0x1d3/0x740 -[ 248.365083] ? finish_task_switch+0x1aa/0x740 -[ 248.369558] ? copy_overflow+0x20/0x20 -[ 248.373437] ? __schedule+0x8f3/0x2060 -[ 248.377296] ? check_noncircular+0x20/0x20 -[ 248.381527] ? find_held_lock+0x39/0x1d0 -[ 248.385573] ? find_held_lock+0x39/0x1d0 -[ 248.389625] ? lock_downgrade+0x980/0x980 -[ 248.393751] ? default_wake_function+0x30/0x50 -[ 248.398320] ? __schedule+0x2060/0x2060 -[ 248.402263] ? do_wait_intr+0x3e0/0x3e0 -[ 248.406215] ? do_raw_spin_trylock+0x190/0x190 -[ 248.410772] ? _raw_spin_unlock_irqrestore+0x31/0xba -[ 248.415852] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 248.420842] ? trace_hardirqs_on+0xd/0x10 -[ 248.424963] ? __kthread_parkme+0x175/0x240 [ 248.429263] kthread+0x37a/0x440 -[ 248.432602] ? process_one_work+0x1be0/0x1be0 -[ 248.437067] ? kthread_stop+0x7b0/0x7b0 [ 248.441018] ret_from_fork+0x24/0x30 [ 248.444727] Code: 00 00 00 fc ff df 44 89 bd 50 ff ff ff 48 c1 e8 03 4c 01 e8 41 83 e7 01 c6 00 f8 74 4e 49 89 c7 49 83 c4 18 e8 4e 25 0b 00 f3 90 <4c> 89 e2 41 c6 07 04 48 c1 ea 03 42 0f b6 14 2a 84 d2 74 09 80 [ 248.463942] Kernel panic - not syncing: softlockup: hung tasks @@ -111,33 +43,11 @@ TITLE: BUG: soft lockup in smp_call_function_many [ 248.492919] Call Trace: [ 248.495472] <IRQ> [ 248.497597] dump_stack+0x194/0x257 -[ 248.501202] ? arch_local_irq_restore+0x53/0x53 -[ 248.505843] ? vprintk_default+0x28/0x30 -[ 248.509878] ? vprintk_func+0x5e/0xc0 -[ 248.513650] ? vsnprintf+0x1ed/0x1900 [ 248.517432] panic+0x1e4/0x41c -[ 248.520597] ? refcount_error_report+0x214/0x214 -[ 248.525340] ? watchdog_timer_fn+0x303/0x320 [ 248.529731] watchdog_timer_fn+0x314/0x320 [ 248.533947] __hrtimer_run_queues+0x349/0xe10 -[ 248.538418] ? lock_downgrade+0x980/0x980 -[ 248.542540] ? watchdog+0x30/0x30 -[ 248.545971] ? hrtimer_cancel+0x40/0x40 -[ 248.549931] ? pvclock_read_flags+0x160/0x160 -[ 248.554417] ? kvm_clock_get_cycles+0x25/0x30 -[ 248.558883] ? ktime_get_update_offsets_now+0x34a/0x520 -[ 248.564229] ? do_timer+0x50/0x50 -[ 248.567661] ? native_apic_msr_write+0x5c/0x80 -[ 248.572213] ? do_raw_spin_trylock+0x190/0x190 -[ 248.576767] ? lapic_next_event+0x54/0x80 -[ 248.580891] ? clockevents_program_event+0x108/0x2e0 [ 248.585978] hrtimer_interrupt+0x1d4/0x5f0 [ 248.590213] smp_apic_timer_interrupt+0x14a/0x700 -[ 248.595032] ? smp_call_function_single_interrupt+0x640/0x640 -[ 248.600889] ? _raw_spin_lock+0x32/0x40 -[ 248.604847] ? handle_edge_irq+0x2b4/0x7c0 -[ 248.609054] ? task_prio+0x40/0x40 -[ 248.612583] ? trace_hardirqs_off_thunk+0x1a/0x1c [ 248.617409] apic_timer_interrupt+0x9d/0xb0 [ 248.621700] </IRQ> [ 248.623909] RIP: 0010:smp_call_function_single+0x364/0x560 @@ -147,87 +57,18 @@ TITLE: BUG: soft lockup in smp_call_function_many [ 248.651662] RBP: ffff8801d2ac7050 R08: 1ffff1003a558dff R09: 0000000000000000 [ 248.658903] R10: ffff8801d2ac7078 R11: 0000000000000000 R12: ffff8801d2ac6ff8 [ 248.666144] R13: dffffc0000000000 R14: 0000000000000000 R15: ffffed003a558df4 -[ 248.673405] ? smp_call_function_single+0x362/0x560 -[ 248.678409] ? add_nops+0x90/0x90 -[ 248.681839] ? generic_exec_single+0x5b0/0x5b0 -[ 248.686393] ? __mutex_lock+0x16f/0x1a80 -[ 248.690427] ? arch_jump_label_transform+0x1b/0x40 -[ 248.695329] ? print_usage_bug+0x3f0/0x3f0 -[ 248.699548] ? _find_next_bit+0xee/0x120 -[ 248.703586] ? add_nops+0x90/0x90 -[ 248.707013] ? find_next_bit+0x27/0x30 [ 248.710884] smp_call_function_many+0x773/0x930 -[ 248.715539] ? smp_call_function_many+0x773/0x930 -[ 248.720362] ? add_nops+0x90/0x90 -[ 248.723796] ? smp_call_function_single_async+0x2d0/0x2d0 -[ 248.729309] ? mark_held_locks+0xb2/0x100 -[ 248.733431] ? text_poke+0x336/0x530 -[ 248.737120] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 248.742109] ? apic_has_pending_timer+0x80/0x2a0 -[ 248.746838] ? trace_hardirqs_on+0xd/0x10 -[ 248.750957] ? add_nops+0x90/0x90 -[ 248.754383] ? apic_has_pending_timer+0x80/0x2a0 [ 248.759110] on_each_cpu+0x3d/0x1b0 -[ 248.762710] ? apic_has_pending_timer+0x7f/0x2a0 [ 248.767443] text_poke_bp+0xe4/0x170 -[ 248.771131] ? __mutex_unlock_slowpath+0xe9/0xac0 -[ 248.775946] ? poke_int3_handler+0x90/0x90 -[ 248.780158] ? wait_for_completion+0x7b0/0x7b0 -[ 248.784721] ? apic_has_pending_timer+0x7f/0x2a0 [ 248.789452] __jump_label_transform.isra.0+0x6a5/0x8a0 -[ 248.794707] ? bug_at+0x20/0x20 -[ 248.797964] ? check_noncircular+0x20/0x20 -[ 248.802170] ? perf_trace_lock_acquire+0xe3/0x980 -[ 248.806990] ? modules_open+0xa0/0xa0 [ 248.810776] arch_jump_label_transform+0x2f/0x40 [ 248.815507] __jump_label_update+0x207/0x2d0 -[ 248.819895] ? static_key_count+0x110/0x110 -[ 248.824192] ? atomic_dec_and_mutex_lock+0x112/0x150 -[ 248.829269] ? mutex_spin_on_owner+0xb50/0xb50 [ 248.833832] jump_label_update+0x22c/0x2b0 [ 248.838048] static_key_slow_dec_cpuslocked+0x176/0x1d0 -[ 248.843387] ? static_key_slow_inc+0x3c0/0x3c0 -[ 248.847951] ? rcu_read_lock_sched_held+0x108/0x120 [ 248.853695] jump_label_update_timeout+0x1f/0x30 [ 248.858424] process_one_work+0xbfd/0x1be0 -[ 248.862653] ? pwq_dec_nr_in_flight+0x450/0x450 -[ 248.867292] ? finish_task_switch+0x1d3/0x740 -[ 248.871756] ? finish_task_switch+0x1aa/0x740 -[ 248.876251] ? perf_trace_lock_acquire+0xe3/0x980 -[ 248.881080] ? perf_trace_lock+0x900/0x900 -[ 248.885287] ? __sched_text_start+0x8/0x8 -[ 248.889411] ? lock_downgrade+0x980/0x980 -[ 248.893533] ? __wake_up_common_lock+0x190/0x310 -[ 248.898281] ? find_held_lock+0x39/0x1d0 -[ 248.902335] ? lock_acquire+0x1d5/0x580 -[ 248.906283] ? worker_thread+0x4a3/0x1990 -[ 248.910414] ? lock_release+0xda0/0xda0 -[ 248.914363] ? worker_pool_assign_id+0x1b0/0x1b0 -[ 248.919094] ? do_raw_spin_trylock+0x190/0x190 [ 248.923671] worker_thread+0x223/0x1990 -[ 248.927650] ? process_one_work+0x1be0/0x1be0 -[ 248.932122] ? _raw_spin_unlock_irq+0x27/0x70 -[ 248.936593] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 248.941582] ? trace_hardirqs_on+0xd/0x10 -[ 248.945708] ? finish_task_switch+0x1d3/0x740 -[ 248.950177] ? finish_task_switch+0x1aa/0x740 -[ 248.954652] ? copy_overflow+0x20/0x20 -[ 248.958530] ? __schedule+0x8f3/0x2060 -[ 248.962386] ? check_noncircular+0x20/0x20 -[ 248.966615] ? find_held_lock+0x39/0x1d0 -[ 248.970661] ? find_held_lock+0x39/0x1d0 -[ 248.974716] ? lock_downgrade+0x980/0x980 -[ 248.978841] ? default_wake_function+0x30/0x50 -[ 248.983411] ? __schedule+0x2060/0x2060 -[ 248.987355] ? do_wait_intr+0x3e0/0x3e0 -[ 248.991308] ? do_raw_spin_trylock+0x190/0x190 -[ 248.995865] ? _raw_spin_unlock_irqrestore+0x31/0xba -[ 249.000944] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 249.005933] ? trace_hardirqs_on+0xd/0x10 -[ 249.010055] ? __kthread_parkme+0x175/0x240 [ 249.014357] kthread+0x37a/0x440 -[ 249.017695] ? process_one_work+0x1be0/0x1be0 -[ 249.022163] ? kthread_stop+0x7b0/0x7b0 [ 249.026113] ret_from_fork+0x24/0x30 [ 249.030488] Dumping ftrace buffer: [ 249.034111] (ftrace buffer empty) diff --git a/pkg/report/testdata/linux/report/67 b/pkg/report/testdata/linux/report/67 index 2eef4bbe5..0976cb34e 100644 --- a/pkg/report/testdata/linux/report/67 +++ b/pkg/report/testdata/linux/report/67 @@ -1,4 +1,4 @@ -TITLE: BUG: soft lockup in mulaw_decode +TITLE: BUG: soft lockup in snd_pcm_oss_write [ 562.725743] watchdog: BUG: soft lockup - CPU#0 stuck for 135s! [syzkaller670324:3527] [ 562.733767] Modules linked in: |