pkg/host: improve KMEMLEAK support

Rewind kmemleak fd before reading it second time,
otherwise we will read truncated reports.

Auto-learn what leak reports we've already seen
and ignore them in future. This is required because
there are some false positives and some fire too frequently.
So now we will hit each leak only once per manager run,
but we still will try to reproduce them.

7 files changed, 111 insertions, 100 deletions

diff --git a/pkg/host/host.go b/pkg/host/host.go
index b23dd9ab2..36c42de65 100644
--- a/pkg/host/host.go
+++ b/pkg/host/host.go
@@ -65,7 +65,7 @@ type Features [numFeatures]Feature
 
 var checkFeature [numFeatures]func() string
 var setupFeature [numFeatures]func() error
-var callbFeature [numFeatures]func()
+var callbFeature [numFeatures]func(leakFrames [][]byte)
 
 func unconditionallyEnabled() string { return "" }
 
@@ -104,11 +104,11 @@ func Check(target *prog.Target) (*Features, error) {
 
 // Setup enables and does any one-time setup for the requested features on the host.
 // Note: this can be called multiple times and must be idempotent.
-func Setup(target *prog.Target, features *Features) (func(), error) {
+func Setup(target *prog.Target, features *Features) (func(leakFrames [][]byte), error) {
 	if target.OS == "akaros" || target.OS == "test" {
 		return nil, nil
 	}
-	var callback func()
+	var callback func([][]byte)
 	for n, setup := range setupFeature {
 		if setup == nil || !features[n].Enabled {
 			continue
@@ -117,15 +117,15 @@ func Setup(target *prog.Target, features *Features) (func(), error) {
 			return nil, err
 		}
 		cb := callbFeature[n]
-		if cb != nil {
-			prev := callback
-			callback = func() {
-				cb()
-				if prev != nil {
-					prev()
-				}
+		if cb == nil {
+			continue
+		}
+		prev := callback
+		callback = func(leakFrames [][]byte) {
+			cb(leakFrames)
+			if prev != nil {
+				prev(leakFrames)
 			}
-
 		}
 	}
 	return callback, nil
diff --git a/pkg/host/host_linux.go b/pkg/host/host_linux.go
index 8214b35af..5c5db793b 100644
--- a/pkg/host/host_linux.go
+++ b/pkg/host/host_linux.go
@@ -469,7 +469,7 @@ func setupLeakChecking() error {
 	return nil
 }
 
-func callbackLeakChecking() {
+func callbackLeakChecking(leakFrames [][]byte) {
 	start := time.Now()
 	fd, err := syscall.Open("/sys/kernel/debug/kmemleak", syscall.O_RDWR, 0)
 	if err != nil {
@@ -506,11 +506,15 @@ func callbackLeakChecking() {
 		if _, err := syscall.Write(fd, []byte("scan")); err != nil {
 			panic(err)
 		}
+		if _, err := syscall.Seek(fd, 0, 0); err != nil {
+			panic(err)
+		}
 		n, err := syscall.Read(fd, buf)
 		if err != nil {
 			panic(err)
 		}
 		nleaks := 0
+	nextLeak:
 		for buf = buf[:n]; len(buf) != 0; {
 			end := bytes.Index(buf[1:], []byte("unreferenced object"))
 			if end != -1 {
@@ -520,14 +524,18 @@ func callbackLeakChecking() {
 			}
 			report := buf[:end]
 			buf = buf[end:]
-			if kmemleakIgnore(report) {
-				continue
+			for _, frame := range leakFrames {
+				if bytes.Contains(report, frame) {
+					continue nextLeak
+				}
 			}
 			// BUG in output should be recognized by manager.
 			fmt.Printf("BUG: memory leak\n%s\n", report)
 			nleaks++
 		}
 		if nleaks != 0 {
+			// If we exit right away, dying executors will dump lots of garbage to console.
+			time.Sleep(time.Hour)
 			os.Exit(1)
 		}
 	}
@@ -536,30 +544,6 @@ func callbackLeakChecking() {
 	}
 }
 
-func kmemleakIgnore(report []byte) bool {
-	// kmemleak has a bunch of false positives (at least what looks like
-	// false positives at first glance). So we are conservative with what we report.
-	// First, we filter out any allocations that don't come from executor processes.
-	// Second, we ignore a bunch of functions entirely.
-	// Ideally, someone should debug/fix all these cases and remove ignores.
-	if !bytes.Contains(report, []byte(`comm "syz-executor`)) {
-		return true
-	}
-	for _, ignore := range []string{
-		" copy_process",
-		" do_execveat_common",
-		" __ext4_",
-		" get_empty_filp",
-		" do_filp_open",
-		" new_inode",
-	} {
-		if bytes.Contains(report, []byte(ignore)) {
-			return true
-		}
-	}
-	return false
-}
-
 func checkSandboxNamespace() string {
 	if err := osutil.IsAccessible("/proc/self/ns/user"); err != nil {
 		return err.Error()
diff --git a/pkg/ipc/ipcconfig/ipcconfig.go b/pkg/ipc/ipcconfig/ipcconfig.go
index 021978274..b94579102 100644
--- a/pkg/ipc/ipcconfig/ipcconfig.go
+++ b/pkg/ipc/ipcconfig/ipcconfig.go
@@ -33,18 +33,11 @@ func Default(target *prog.Target) (*ipc.Config, *ipc.ExecOpts, error) {
 	if *flagDebug {
 		c.Flags |= ipc.FlagDebug
 	}
-	switch *flagSandbox {
-	case "none":
-	case "setuid":
-		c.Flags |= ipc.FlagSandboxSetuid
-	case "namespace":
-		c.Flags |= ipc.FlagSandboxNamespace
-	case "android_untrusted_app":
-		c.Flags |= ipc.FlagSandboxAndroidUntrustedApp
-	default:
-		return nil, nil, fmt.Errorf("flag sandbox must contain one of none/setuid/namespace/android_untrusted_app")
+	sandboxFlags, err := SandboxToFlags(*flagSandbox)
+	if err != nil {
+		return nil, nil, err
 	}
-
+	c.Flags |= sandboxFlags
 	sysTarget := targets.Get(target.OS, target.Arch)
 	if sysTarget.ExecutorUsesShmem {
 		c.Flags |= ipc.FlagUseShmem
@@ -65,3 +58,29 @@ func Default(target *prog.Target) (*ipc.Config, *ipc.ExecOpts, error) {
 
 	return c, opts, nil
 }
+
+func SandboxToFlags(sandbox string) (ipc.EnvFlags, error) {
+	switch sandbox {
+	case "none":
+		return 0, nil
+	case "setuid":
+		return ipc.FlagSandboxSetuid, nil
+	case "namespace":
+		return ipc.FlagSandboxNamespace, nil
+	case "android_untrusted_app":
+		return ipc.FlagSandboxAndroidUntrustedApp, nil
+	default:
+		return 0, fmt.Errorf("sandbox must contain one of none/setuid/namespace/android_untrusted_app")
+	}
+}
+
+func FlagsToSandbox(flags ipc.EnvFlags) string {
+	if flags&ipc.FlagSandboxSetuid != 0 {
+		return "setuid"
+	} else if flags&ipc.FlagSandboxNamespace != 0 {
+		return "namespace"
+	} else if flags&ipc.FlagSandboxAndroidUntrustedApp != 0 {
+		return "android_untrusted_app"
+	}
+	return "none"
+}
diff --git a/pkg/rpctype/rpctype.go b/pkg/rpctype/rpctype.go
index eb2303950..68f85eedc 100644
--- a/pkg/rpctype/rpctype.go
+++ b/pkg/rpctype/rpctype.go
@@ -29,11 +29,12 @@ type ConnectArgs struct {
 }
 
 type ConnectRes struct {
-	EnabledCalls   []int
-	GitRevision    string
-	TargetRevision string
-	AllSandboxes   bool
-	CheckResult    *CheckArgs
+	EnabledCalls     []int
+	GitRevision      string
+	TargetRevision   string
+	AllSandboxes     bool
+	CheckResult      *CheckArgs
+	MemoryLeakFrames [][]byte
 }
 
 type CheckArgs struct {
diff --git a/pkg/runtest/run.go b/pkg/runtest/run.go
index 115f5ecb7..0ec0bb125 100644
--- a/pkg/runtest/run.go
+++ b/pkg/runtest/run.go
@@ -26,6 +26,7 @@ import (
 	"github.com/google/syzkaller/pkg/csource"
 	"github.com/google/syzkaller/pkg/host"
 	"github.com/google/syzkaller/pkg/ipc"
+	"github.com/google/syzkaller/pkg/ipc/ipcconfig"
 	"github.com/google/syzkaller/pkg/osutil"
 	"github.com/google/syzkaller/prog"
 	"github.com/google/syzkaller/sys/targets"
@@ -307,14 +308,11 @@ func (ctx *Context) createSyzTest(p *prog.Prog, sandbox string, threaded, cov bo
 	if sysTarget.ExecutorUsesForkServer {
 		cfg.Flags |= ipc.FlagUseForkServer
 	}
-	switch sandbox {
-	case "namespace":
-		cfg.Flags |= ipc.FlagSandboxNamespace
-	case "setuid":
-		cfg.Flags |= ipc.FlagSandboxSetuid
-	case "android_untrusted_app":
-		cfg.Flags |= ipc.FlagSandboxAndroidUntrustedApp
+	sandboxFlags, err := ipcconfig.SandboxToFlags(sandbox)
+	if err != nil {
+		return nil, err
 	}
+	cfg.Flags |= sandboxFlags
 	if threaded {
 		opts.Flags |= ipc.FlagThreaded | ipc.FlagCollide
 	}
diff --git a/syz-fuzzer/fuzzer.go b/syz-fuzzer/fuzzer.go
index cb259a914..fce4b48b8 100644
--- a/syz-fuzzer/fuzzer.go
+++ b/syz-fuzzer/fuzzer.go
@@ -116,15 +116,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("failed to create default ipc config: %v", err)
 	}
-	sandbox := "none"
-	if config.Flags&ipc.FlagSandboxSetuid != 0 {
-		sandbox = "setuid"
-	} else if config.Flags&ipc.FlagSandboxNamespace != 0 {
-		sandbox = "namespace"
-	} else if config.Flags&ipc.FlagSandboxAndroidUntrustedApp != 0 {
-		sandbox = "android_untrusted_app"
-	}
-
+	sandbox := ipcconfig.FlagsToSandbox(config.Flags)
 	shutdown := make(chan struct{})
 	osutil.HandleInterrupts(shutdown)
 	go func() {
@@ -191,6 +183,10 @@ func main() {
 	if err != nil {
 		log.Fatalf("BUG: %v", err)
 	}
+	var gateCallback func()
+	if periodicCallback != nil {
+		gateCallback = func() { periodicCallback(r.MemoryLeakFrames) }
+	}
 	if r.CheckResult.Features[host.FeatureNetworkInjection].Enabled {
 		config.Flags |= ipc.FlagEnableTun
 	}
@@ -213,7 +209,7 @@ func main() {
 		outputType:               outputType,
 		config:                   config,
 		execOpts:                 execOpts,
-		gate:                     ipc.NewGate(2**flagProcs, periodicCallback),
+		gate:                     ipc.NewGate(2**flagProcs, gateCallback),
 		workQueue:                newWorkQueue(*flagProcs, needPoll),
 		needPoll:                 needPoll,
 		manager:                  manager,
diff --git a/syz-manager/manager.go b/syz-manager/manager.go
index 57e2e5073..09f27d657 100644
--- a/syz-manager/manager.go
+++ b/syz-manager/manager.go
@@ -13,6 +13,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -70,15 +71,16 @@ type Manager struct {
 	phase           int
 	enabledSyscalls []int
 
-	candidates     []rpctype.RPCCandidate // untriaged inputs from corpus and hub
-	disabledHashes map[string]struct{}
-	corpus         map[string]rpctype.RPCInput
-	corpusCover    cover.Cover
-	corpusSignal   signal.Signal
-	maxSignal      signal.Signal
-	prios          [][]float32
-	newRepros      [][]byte
-	lastMinCorpus  int
+	candidates       []rpctype.RPCCandidate // untriaged inputs from corpus and hub
+	disabledHashes   map[string]struct{}
+	corpus           map[string]rpctype.RPCInput
+	corpusCover      cover.Cover
+	corpusSignal     signal.Signal
+	maxSignal        signal.Signal
+	prios            [][]float32
+	newRepros        [][]byte
+	lastMinCorpus    int
+	memoryLeakFrames map[string]bool
 
 	fuzzers        map[string]*Fuzzer
 	needMoreRepros chan chan bool
@@ -171,26 +173,27 @@ func RunManager(cfg *mgrconfig.Config, target *prog.Target, sysTarget *targets.T
 	}
 
 	mgr := &Manager{
-		cfg:             cfg,
-		vmPool:          vmPool,
-		target:          target,
-		sysTarget:       sysTarget,
-		reporter:        reporter,
-		crashdir:        crashdir,
-		startTime:       time.Now(),
-		stats:           new(Stats),
-		fuzzerStats:     make(map[string]uint64),
-		crashTypes:      make(map[string]bool),
-		enabledSyscalls: enabledSyscalls,
-		corpus:          make(map[string]rpctype.RPCInput),
-		disabledHashes:  make(map[string]struct{}),
-		fuzzers:         make(map[string]*Fuzzer),
-		fresh:           true,
-		vmStop:          make(chan bool),
-		hubReproQueue:   make(chan *Crash, 10),
-		needMoreRepros:  make(chan chan bool),
-		reproRequest:    make(chan chan map[string]bool),
-		usedFiles:       make(map[string]time.Time),
+		cfg:              cfg,
+		vmPool:           vmPool,
+		target:           target,
+		sysTarget:        sysTarget,
+		reporter:         reporter,
+		crashdir:         crashdir,
+		startTime:        time.Now(),
+		stats:            new(Stats),
+		fuzzerStats:      make(map[string]uint64),
+		crashTypes:       make(map[string]bool),
+		enabledSyscalls:  enabledSyscalls,
+		corpus:           make(map[string]rpctype.RPCInput),
+		disabledHashes:   make(map[string]struct{}),
+		memoryLeakFrames: make(map[string]bool),
+		fuzzers:          make(map[string]*Fuzzer),
+		fresh:            true,
+		vmStop:           make(chan bool),
+		hubReproQueue:    make(chan *Crash, 10),
+		needMoreRepros:   make(chan chan bool),
+		reproRequest:     make(chan chan map[string]bool),
+		usedFiles:        make(map[string]time.Time),
 	}
 
 	log.Logf(0, "loading corpus...")
@@ -602,6 +605,12 @@ func (mgr *Manager) emailCrash(crash *Crash) {
 }
 
 func (mgr *Manager) saveCrash(crash *Crash) bool {
+	if strings.HasPrefix(crash.Title, report.MemoryLeakPrefix) {
+		frame := crash.Title[len(report.MemoryLeakPrefix):]
+		mgr.mu.Lock()
+		mgr.memoryLeakFrames[frame] = true
+		mgr.mu.Unlock()
+	}
 	if crash.Suppressed {
 		log.Logf(0, "vm-%v: suppressed crash %v", crash.vmIndex, crash.Title)
 		mgr.stats.crashSuppressed.inc()
@@ -910,6 +919,10 @@ func (mgr *Manager) Connect(a *rpctype.ConnectArgs, r *rpctype.ConnectRes) error
 	for _, inp := range mgr.corpus {
 		f.inputs = append(f.inputs, inp)
 	}
+	r.MemoryLeakFrames = make([][]byte, 0, len(mgr.memoryLeakFrames))
+	for frame := range mgr.memoryLeakFrames {
+		r.MemoryLeakFrames = append(r.MemoryLeakFrames, []byte(frame))
+	}
 	r.EnabledCalls = mgr.enabledSyscalls
 	r.CheckResult = mgr.checkResult
 	r.GitRevision = sys.GitRevision