aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAleksandr Nogikh <nogikh@google.com>2024-04-30 14:57:05 +0200
committerAleksandr Nogikh <nogikh@google.com>2024-05-02 17:00:15 +0000
commitf61ed74922653fa99fbc3f915d36f6741e2df41c (patch)
tree8e54d89ab30294b1b03d42e886cf9ce61d5e5e14
parent22ee48a2879809608f79cc23c914859fa2335d59 (diff)
syz-manager: abort fuzzing immediately after crash detection
If the crash did not result in a panic, we continue normal fuzzing operation for several more seconds until a delay has passed and the VM was stopped. It results in 20-30 more programs that are: 1) Run on a potentially corrupted machine. 2) Litter the logs -- these programs are definitely not related to the crash. 3) Make it trickier to automatically track crash risk for individual syscalls/programs (if we get to this). Add an option to vm.Run that allows to determine a crash immediatey after it was detected. Use it to stop program exchange and program injection in rpc.go.
-rw-r--r--syz-manager/manager.go12
-rw-r--r--syz-manager/rpc.go23
-rw-r--r--vm/vm.go12
3 files changed, 40 insertions, 7 deletions
diff --git a/syz-manager/manager.go b/syz-manager/manager.go
index ffc1b87d6..3e77e7885 100644
--- a/syz-manager/manager.go
+++ b/syz-manager/manager.go
@@ -727,13 +727,12 @@ func (mgr *Manager) runInstance(index int) (*Crash, error) {
if fuzzer := mgr.fuzzer.Load(); fuzzer != nil {
maxSignal = fuzzer.Cover.CopyMaxSignal()
}
- // Use unique instance names to keep name collisions in case of untimely RPC messages.
+ // Use unique instance names to prevent name collisions in case of untimely RPC messages.
instanceName := fmt.Sprintf("vm-%d", mgr.nextInstanceID.Add(1))
injectLog := make(chan []byte, 10)
mgr.serv.createInstance(instanceName, maxSignal, injectLog)
rep, vmInfo, err := mgr.runInstanceInner(index, instanceName, injectLog)
-
machineInfo := mgr.serv.shutdownInstance(instanceName, rep != nil)
if len(vmInfo) != 0 {
machineInfo = append(append(vmInfo, '\n'), machineInfo...)
@@ -822,7 +821,14 @@ func (mgr *Manager) runInstanceInner(index int, instanceName string, injectLog <
}
cmd := instance.FuzzerCmd(args)
_, rep, err := inst.Run(mgr.cfg.Timeouts.VMRunningTime, mgr.reporter, cmd,
- vm.ExitTimeout, vm.StopChan(mgr.vmStop), vm.InjectOutput(injectLog))
+ vm.ExitTimeout, vm.StopChan(mgr.vmStop), vm.InjectOutput(injectLog),
+ vm.EarlyFinishCb(func() {
+ // Depending on the crash type and kernel config, fuzzing may continue
+ // running for several seconds even after kernel has printed a crash report.
+ // This litters the log and we want to prevent it.
+ mgr.serv.stopFuzzing(instanceName)
+ }),
+ )
if err != nil {
return nil, nil, fmt.Errorf("failed to run fuzzer: %w", err)
}
diff --git a/syz-manager/rpc.go b/syz-manager/rpc.go
index 10b76d0bb..41b29ca3a 100644
--- a/syz-manager/rpc.go
+++ b/syz-manager/rpc.go
@@ -67,9 +67,10 @@ type RPCServer struct {
}
type Runner struct {
- name string
- injectLog chan<- []byte
- injectStop chan bool
+ name string
+ injectLog chan<- []byte
+ injectStop chan bool
+ stopFuzzing atomic.Bool
machineInfo []byte
instModules *cover.CanonicalizerInstance
@@ -410,7 +411,11 @@ func (serv *RPCServer) ExchangeInfo(a *rpctype.ExchangeInfoRequest, r *rpctype.E
func (serv *RPCServer) findRunner(name string) *Runner {
if val, _ := serv.runners.Load(name); val != nil {
- return val.(*Runner)
+ runner := val.(*Runner)
+ if runner.stopFuzzing.Load() {
+ return nil
+ }
+ return runner
}
// There might be a parallel shutdownInstance().
// Ignore requests then.
@@ -430,6 +435,16 @@ func (serv *RPCServer) createInstance(name string, maxSignal signal.Signal, inje
}
}
+// stopInstance prevents further request exchange requests.
+// To make RPCServer fully forget an instance, shutdownInstance() must be called.
+func (serv *RPCServer) stopFuzzing(name string) {
+ runner := serv.findRunner(name)
+ if runner == nil {
+ return
+ }
+ runner.stopFuzzing.Store(true)
+}
+
func (serv *RPCServer) shutdownInstance(name string, crashed bool) []byte {
if !serv.checkDone.Load() {
log.Fatalf("VM is exited while checking is not done")
diff --git a/vm/vm.go b/vm/vm.go
index 5972771b7..351163f2e 100644
--- a/vm/vm.go
+++ b/vm/vm.go
@@ -200,6 +200,9 @@ type StopChan <-chan bool
type InjectOutput <-chan []byte
type OutputSize int
+// An early notification that the command has finished / VM crashed.
+type EarlyFinishCb func()
+
// Run runs cmd inside of the VM (think of ssh cmd) and monitors command execution
// and the kernel console output. It detects kernel oopses in output, lost connections, hangs, etc.
// Returns command+kernel output and a non-symbolized crash report (nil if no error happens).
@@ -212,6 +215,7 @@ func (inst *Instance) Run(timeout time.Duration, reporter *report.Reporter, comm
exit := ExitNormal
var stop <-chan bool
var injected <-chan []byte
+ var finished func()
outputSize := beforeContextDefault
for _, o := range opts {
switch opt := o.(type) {
@@ -223,6 +227,8 @@ func (inst *Instance) Run(timeout time.Duration, reporter *report.Reporter, comm
outputSize = int(opt)
case InjectOutput:
injected = (<-chan []byte)(opt)
+ case EarlyFinishCb:
+ finished = opt
default:
panic(fmt.Sprintf("unknown option %#v", opt))
}
@@ -236,6 +242,7 @@ func (inst *Instance) Run(timeout time.Duration, reporter *report.Reporter, comm
outc: outc,
injected: injected,
errc: errc,
+ finished: finished,
reporter: reporter,
beforeContext: outputSize,
exit: exit,
@@ -281,6 +288,7 @@ type monitor struct {
inst *Instance
outc <-chan []byte
injected <-chan []byte
+ finished func()
errc <-chan error
reporter *report.Reporter
exit ExitCondition
@@ -378,6 +386,10 @@ func (mon *monitor) appendOutput(out []byte) *report.Report {
}
func (mon *monitor) extractError(defaultError string) *report.Report {
+ if mon.finished != nil {
+ // If the caller wanted an early notification, provide it.
+ mon.finished()
+ }
diagOutput, diagWait := []byte{}, false
if defaultError != "" {
diagOutput, diagWait = mon.inst.diagnose(mon.createReport(defaultError))