From f61ed74922653fa99fbc3f915d36f6741e2df41c Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Tue, 30 Apr 2024 14:57:05 +0200 Subject: syz-manager: abort fuzzing immediately after crash detection If the crash did not result in a panic, we continue normal fuzzing operation for several more seconds until a delay has passed and the VM was stopped. It results in 20-30 more programs that are: 1) Run on a potentially corrupted machine. 2) Litter the logs -- these programs are definitely not related to the crash. 3) Make it trickier to automatically track crash risk for individual syscalls/programs (if we get to this). Add an option to vm.Run that allows to determine a crash immediatey after it was detected. Use it to stop program exchange and program injection in rpc.go. --- syz-manager/manager.go | 12 +++++++++--- syz-manager/rpc.go | 23 +++++++++++++++++++---- vm/vm.go | 12 ++++++++++++ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/syz-manager/manager.go b/syz-manager/manager.go index ffc1b87d6..3e77e7885 100644 --- a/syz-manager/manager.go +++ b/syz-manager/manager.go @@ -727,13 +727,12 @@ func (mgr *Manager) runInstance(index int) (*Crash, error) { if fuzzer := mgr.fuzzer.Load(); fuzzer != nil { maxSignal = fuzzer.Cover.CopyMaxSignal() } - // Use unique instance names to keep name collisions in case of untimely RPC messages. + // Use unique instance names to prevent name collisions in case of untimely RPC messages. instanceName := fmt.Sprintf("vm-%d", mgr.nextInstanceID.Add(1)) injectLog := make(chan []byte, 10) mgr.serv.createInstance(instanceName, maxSignal, injectLog) rep, vmInfo, err := mgr.runInstanceInner(index, instanceName, injectLog) - machineInfo := mgr.serv.shutdownInstance(instanceName, rep != nil) if len(vmInfo) != 0 { machineInfo = append(append(vmInfo, '\n'), machineInfo...) @@ -822,7 +821,14 @@ func (mgr *Manager) runInstanceInner(index int, instanceName string, injectLog < } cmd := instance.FuzzerCmd(args) _, rep, err := inst.Run(mgr.cfg.Timeouts.VMRunningTime, mgr.reporter, cmd, - vm.ExitTimeout, vm.StopChan(mgr.vmStop), vm.InjectOutput(injectLog)) + vm.ExitTimeout, vm.StopChan(mgr.vmStop), vm.InjectOutput(injectLog), + vm.EarlyFinishCb(func() { + // Depending on the crash type and kernel config, fuzzing may continue + // running for several seconds even after kernel has printed a crash report. + // This litters the log and we want to prevent it. + mgr.serv.stopFuzzing(instanceName) + }), + ) if err != nil { return nil, nil, fmt.Errorf("failed to run fuzzer: %w", err) } diff --git a/syz-manager/rpc.go b/syz-manager/rpc.go index 10b76d0bb..41b29ca3a 100644 --- a/syz-manager/rpc.go +++ b/syz-manager/rpc.go @@ -67,9 +67,10 @@ type RPCServer struct { } type Runner struct { - name string - injectLog chan<- []byte - injectStop chan bool + name string + injectLog chan<- []byte + injectStop chan bool + stopFuzzing atomic.Bool machineInfo []byte instModules *cover.CanonicalizerInstance @@ -410,7 +411,11 @@ func (serv *RPCServer) ExchangeInfo(a *rpctype.ExchangeInfoRequest, r *rpctype.E func (serv *RPCServer) findRunner(name string) *Runner { if val, _ := serv.runners.Load(name); val != nil { - return val.(*Runner) + runner := val.(*Runner) + if runner.stopFuzzing.Load() { + return nil + } + return runner } // There might be a parallel shutdownInstance(). // Ignore requests then. @@ -430,6 +435,16 @@ func (serv *RPCServer) createInstance(name string, maxSignal signal.Signal, inje } } +// stopInstance prevents further request exchange requests. +// To make RPCServer fully forget an instance, shutdownInstance() must be called. +func (serv *RPCServer) stopFuzzing(name string) { + runner := serv.findRunner(name) + if runner == nil { + return + } + runner.stopFuzzing.Store(true) +} + func (serv *RPCServer) shutdownInstance(name string, crashed bool) []byte { if !serv.checkDone.Load() { log.Fatalf("VM is exited while checking is not done") diff --git a/vm/vm.go b/vm/vm.go index 5972771b7..351163f2e 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -200,6 +200,9 @@ type StopChan <-chan bool type InjectOutput <-chan []byte type OutputSize int +// An early notification that the command has finished / VM crashed. +type EarlyFinishCb func() + // Run runs cmd inside of the VM (think of ssh cmd) and monitors command execution // and the kernel console output. It detects kernel oopses in output, lost connections, hangs, etc. // Returns command+kernel output and a non-symbolized crash report (nil if no error happens). @@ -212,6 +215,7 @@ func (inst *Instance) Run(timeout time.Duration, reporter *report.Reporter, comm exit := ExitNormal var stop <-chan bool var injected <-chan []byte + var finished func() outputSize := beforeContextDefault for _, o := range opts { switch opt := o.(type) { @@ -223,6 +227,8 @@ func (inst *Instance) Run(timeout time.Duration, reporter *report.Reporter, comm outputSize = int(opt) case InjectOutput: injected = (<-chan []byte)(opt) + case EarlyFinishCb: + finished = opt default: panic(fmt.Sprintf("unknown option %#v", opt)) } @@ -236,6 +242,7 @@ func (inst *Instance) Run(timeout time.Duration, reporter *report.Reporter, comm outc: outc, injected: injected, errc: errc, + finished: finished, reporter: reporter, beforeContext: outputSize, exit: exit, @@ -281,6 +288,7 @@ type monitor struct { inst *Instance outc <-chan []byte injected <-chan []byte + finished func() errc <-chan error reporter *report.Reporter exit ExitCondition @@ -378,6 +386,10 @@ func (mon *monitor) appendOutput(out []byte) *report.Report { } func (mon *monitor) extractError(defaultError string) *report.Report { + if mon.finished != nil { + // If the caller wanted an early notification, provide it. + mon.finished() + } diagOutput, diagWait := []byte{}, false if defaultError != "" { diagOutput, diagWait = mon.inst.diagnose(mon.createReport(defaultError)) -- cgit mrf-deployment