tools: add a syz-diff tool

This is the prototype version of the patch series fuzzing functionality
based on the syzkaller fuzzing engine.

The tool takes two syzkaller configs -- one for the base kernel, one for
the patched kernel. Optionally the patch itself can be also provided.

syz-diff will consider a bug patched-only if:
1) It happened while fuzzing the patched kernel.
2) It was never observed on the base kernel.
3) The tool found a repro on the patched kernel.
4) The repro did not crash the base kernel.

9 files changed, 943 insertions, 14 deletions

diff --git a/Makefile b/Makefile
index f4c94c45e..6b770e7b5 100644
--- a/Makefile
+++ b/Makefile
@@ -170,6 +170,9 @@ repro: descriptions
 mutate: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-mutate github.com/google/syzkaller/tools/syz-mutate
 
+diff: descriptions target
+	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-diff github.com/google/syzkaller/tools/syz-diff
+
 prog2c: descriptions
 	GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(HOSTGO) build $(GOHOSTFLAGS) -o ./bin/syz-prog2c github.com/google/syzkaller/tools/syz-prog2c
 
diff --git a/pkg/fuzzer/queue/queue.go b/pkg/fuzzer/queue/queue.go
index 0a56c76d3..1d25cbc8a 100644
--- a/pkg/fuzzer/queue/queue.go
+++ b/pkg/fuzzer/queue/queue.go
@@ -8,6 +8,7 @@ import (
 	"context"
 	"encoding/gob"
 	"fmt"
+	"math/rand"
 	"sync"
 	"sync/atomic"
 
@@ -459,3 +460,70 @@ func (do *defaultOpts) Next() *Request {
 	req.ExecOpts.SandboxArg = do.opts.SandboxArg
 	return req
 }
+
+// RandomQueue holds up to |size| elements.
+// Next() evicts a random one.
+// On Submit(), if the queue is full, a random element is replaced.
+type RandomQueue struct {
+	mu      sync.Mutex
+	queue   []*Request
+	maxSize int
+	rnd     *rand.Rand
+}
+
+func NewRandomQueue(size int, rnd *rand.Rand) *RandomQueue {
+	return &RandomQueue{
+		maxSize: size,
+		rnd:     rnd,
+	}
+}
+
+func (rq *RandomQueue) Next() *Request {
+	rq.mu.Lock()
+	defer rq.mu.Unlock()
+	if len(rq.queue) == 0 {
+		return nil
+	}
+	pos := rq.rnd.Intn(len(rq.queue))
+	item := rq.queue[pos]
+
+	last := len(rq.queue) - 1
+	rq.queue[pos] = rq.queue[last]
+	rq.queue[last] = nil
+	rq.queue = rq.queue[0 : len(rq.queue)-1]
+	return item
+}
+
+func (rq *RandomQueue) Submit(req *Request) {
+	rq.mu.Lock()
+	defer rq.mu.Unlock()
+	if len(rq.queue) < rq.maxSize {
+		rq.queue = append(rq.queue, req)
+	} else {
+		pos := rq.rnd.Intn(rq.maxSize + 1)
+		if pos < len(rq.queue) {
+			rq.queue[pos].Done(&Result{Status: ExecFailure})
+			rq.queue[pos] = req
+		}
+	}
+}
+
+type tee struct {
+	queue Executor
+	src   Source
+}
+
+func Tee(src Source, queue Executor) Source {
+	return &tee{src: src, queue: queue}
+}
+
+func (t *tee) Next() *Request {
+	req := t.src.Next()
+	if req == nil {
+		return nil
+	}
+	t.queue.Submit(&Request{
+		Prog: req.Prog.Clone(),
+	})
+	return req
+}
diff --git a/pkg/manager/crash.go b/pkg/manager/crash.go
index 5edc9f5a9..56142695a 100644
--- a/pkg/manager/crash.go
+++ b/pkg/manager/crash.go
@@ -306,11 +306,11 @@ func (cs *CrashStore) BugList() ([]*BugInfo, error) {
 	return ret, nil
 }
 
-func (cs *CrashStore) titleToID(title string) string {
+func crashHash(title string) string {
 	sig := hash.Hash([]byte(title))
 	return sig.String()
 }
 
 func (cs *CrashStore) path(title string) string {
-	return filepath.Join(cs.BaseDir, "crashes", cs.titleToID(title))
+	return filepath.Join(cs.BaseDir, "crashes", crashHash(title))
 }
diff --git a/pkg/manager/crash_test.go b/pkg/manager/crash_test.go
index f017f0175..4348dc2f1 100644
--- a/pkg/manager/crash_test.go
+++ b/pkg/manager/crash_test.go
@@ -76,7 +76,7 @@ func TestMaxCrashLogs(t *testing.T) {
 		assert.NoError(t, err)
 	}
 
-	info, err := crashStore.BugInfo(crashStore.titleToID("Title A"), false)
+	info, err := crashStore.BugInfo(crashHash("Title A"), false)
 	assert.NoError(t, err)
 	assert.Len(t, info.Crashes, 5)
 }
@@ -105,7 +105,7 @@ func TestCrashRepro(t *testing.T) {
 	}, []byte("prog text"), []byte("c prog text"))
 	assert.NoError(t, err)
 
-	report, err := crashStore.Report(crashStore.titleToID("Some title"))
+	report, err := crashStore.Report(crashHash("Some title"))
 	assert.NoError(t, err)
 	assert.Equal(t, "Some title", report.Title)
 	assert.Equal(t, "abcd", report.Tag)
diff --git a/pkg/manager/diff.go b/pkg/manager/diff.go
new file mode 100644
index 000000000..76e2b97ff
--- /dev/null
+++ b/pkg/manager/diff.go
@@ -0,0 +1,139 @@
+// Copyright 2024 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package manager
+
+import (
+	"fmt"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/google/syzkaller/pkg/log"
+	"github.com/google/syzkaller/pkg/osutil"
+)
+
+type DiffBug struct {
+	Title   string
+	Base    DiffBugInfo
+	Patched DiffBugInfo
+}
+
+func (bug DiffBug) PatchedOnly() bool {
+	return bug.Base.NotCrashed && bug.Patched.Crashes > 0
+}
+
+func (bug DiffBug) AffectsBoth() bool {
+	return bug.Base.Crashes > 0 && bug.Patched.Crashes > 0
+}
+
+type DiffBugInfo struct {
+	Crashes    int  // Count of detected crashes.
+	NotCrashed bool // If were proven not to crash by running a repro.
+
+	// File paths.
+	Report   string
+	Repro    string
+	ReproLog string
+	CrashLog string
+}
+
+// DiffFuzzerStore provides the functionality of a database of the patch fuzzing.
+type DiffFuzzerStore struct {
+	BasePath string
+
+	mu   sync.Mutex
+	bugs map[string]*DiffBug
+}
+
+func (s *DiffFuzzerStore) BaseCrashed(title string, report []byte) {
+	s.patch(title, func(obj *DiffBug) {
+		obj.Base.Crashes++
+		if len(report) > 0 {
+			obj.Base.Report = s.saveFile(title, "base_report", report)
+		}
+	})
+}
+
+func (s *DiffFuzzerStore) EverCrashedBase(title string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	obj := s.bugs[title]
+	return obj != nil && obj.Base.Crashes > 0
+}
+
+func (s *DiffFuzzerStore) BaseNotCrashed(title string) {
+	s.patch(title, func(obj *DiffBug) {
+		if obj.Base.Crashes == 0 {
+			obj.Base.NotCrashed = true
+		}
+	})
+}
+
+func (s *DiffFuzzerStore) PatchedCrashed(title string, report, log []byte) {
+	s.patch(title, func(obj *DiffBug) {
+		obj.Patched.Crashes++
+		if len(report) > 0 {
+			obj.Patched.Report = s.saveFile(title, "patched_report", report)
+		}
+		if len(log) > 0 && obj.Patched.CrashLog == "" {
+			obj.Patched.CrashLog = s.saveFile(title, "patched_crash_log", log)
+		}
+	})
+}
+
+func (s *DiffFuzzerStore) SaveRepro(result *ReproResult) {
+	title := result.Crash.Report.Title
+	if result.Repro != nil {
+		// If there's a repro, save under the new title.
+		title = result.Repro.Report.Title
+	}
+
+	now := time.Now().Unix()
+	crashLog := fmt.Sprintf("%v.crash.log", now)
+	s.saveFile(title, crashLog, result.Crash.Output)
+	log.Logf(0, "%q: saved crash log into %s", title, crashLog)
+
+	s.patch(title, func(obj *DiffBug) {
+		if result.Repro != nil {
+			obj.Patched.Repro = s.saveFile(title, reproFileName, result.Repro.Prog.Serialize())
+		}
+		if result.Stats != nil {
+			reproLog := fmt.Sprintf("%v.repro.log", now)
+			obj.Patched.ReproLog = s.saveFile(title, reproLog, result.Stats.FullLog())
+			log.Logf(0, "%q: saved repro log into %s", title, reproLog)
+		}
+	})
+}
+
+func (s *DiffFuzzerStore) List() []DiffBug {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	var list []DiffBug
+	for _, obj := range s.bugs {
+		list = append(list, *obj)
+	}
+	return list
+}
+
+func (s *DiffFuzzerStore) saveFile(title, name string, data []byte) string {
+	hash := crashHash(title)
+	path := filepath.Join(s.BasePath, "crashes", hash)
+	osutil.MkdirAll(path)
+	osutil.WriteFile(filepath.Join(path, name), data)
+	return filepath.Join("crashes", hash, name)
+}
+
+func (s *DiffFuzzerStore) patch(title string, cb func(*DiffBug)) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.bugs == nil {
+		s.bugs = map[string]*DiffBug{}
+	}
+	obj, ok := s.bugs[title]
+	if !ok {
+		obj = &DiffBug{Title: title}
+		s.bugs[title] = obj
+	}
+	cb(obj)
+}
diff --git a/pkg/manager/http.go b/pkg/manager/http.go
index 155765534..c94ce727f 100644
--- a/pkg/manager/http.go
+++ b/pkg/manager/http.go
@@ -51,6 +51,7 @@ type HTTPServer struct {
 	StartTime  time.Time
 	Corpus     *corpus.Corpus
 	CrashStore *CrashStore
+	DiffStore  *DiffFuzzerStore
 
 	// Set dynamically.
 	Fuzzer          atomic.Pointer[fuzzer.Fuzzer]
@@ -77,13 +78,15 @@ func (serv *HTTPServer) Serve() {
 	handle("/syscalls", serv.httpSyscalls)
 	handle("/corpus", serv.httpCorpus)
 	handle("/corpus.db", serv.httpDownloadCorpus)
-	handle("/crash", serv.httpCrash)
+	if serv.CrashStore != nil {
+		handle("/crash", serv.httpCrash)
+		handle("/report", serv.httpReport)
+	}
 	handle("/cover", serv.httpCover)
 	handle("/subsystemcover", serv.httpSubsystemCover)
 	handle("/modulecover", serv.httpModuleCover)
 	handle("/prio", serv.httpPrio)
 	handle("/file", serv.httpFile)
-	handle("/report", serv.httpReport)
 	handle("/rawcover", serv.httpRawCover)
 	handle("/rawcoverfiles", serv.httpRawCoverFiles)
 	handle("/filterpcs", serv.httpFilterPCs)
@@ -125,11 +128,15 @@ func (serv *HTTPServer) httpSummary(w http.ResponseWriter, r *http.Request) {
 			Link:  stat.Link,
 		})
 	}
-
-	var err error
-	if data.Crashes, err = serv.collectCrashes(serv.Cfg.Workdir); err != nil {
-		http.Error(w, fmt.Sprintf("failed to collect crashes: %v", err), http.StatusInternalServerError)
-		return
+	if serv.CrashStore != nil {
+		var err error
+		if data.Crashes, err = serv.collectCrashes(serv.Cfg.Workdir); err != nil {
+			http.Error(w, fmt.Sprintf("failed to collect crashes: %v", err), http.StatusInternalServerError)
+			return
+		}
+	}
+	if serv.DiffStore != nil {
+		data.PatchedOnly, data.AffectsBoth, data.InProgress = serv.collectDiffCrashes()
 	}
 	executeTemplate(w, summaryTemplate, data)
 }
@@ -695,15 +702,61 @@ func (serv *HTTPServer) httpFilterPCs(w http.ResponseWriter, r *http.Request) {
 	serv.httpCoverCover(w, r, DoFilterPCs)
 }
 
-func (serv *HTTPServer) collectCrashes(workdir string) ([]*UICrashType, error) {
-	var repros map[string]bool
+func (serv *HTTPServer) collectDiffCrashes() (patchedOnly, both, inProgress *UIDiffTable) {
+	for _, item := range serv.allDiffCrashes() {
+		if item.PatchedOnly() {
+			if patchedOnly == nil {
+				patchedOnly = &UIDiffTable{Title: "Patched-only"}
+			}
+			patchedOnly.List = append(patchedOnly.List, item)
+		} else if item.AffectsBoth() {
+			if both == nil {
+				both = &UIDiffTable{Title: "Affects both"}
+			}
+			both.List = append(both.List, item)
+		} else {
+			if inProgress == nil {
+				inProgress = &UIDiffTable{Title: "In Progress"}
+			}
+			inProgress.List = append(inProgress.List, item)
+		}
+	}
+	return
+}
+
+func (serv *HTTPServer) allDiffCrashes() []UIDiffBug {
+	repros := serv.nowReproducing()
+	var list []UIDiffBug
+	for _, bug := range serv.DiffStore.List() {
+		list = append(list, UIDiffBug{
+			DiffBug:     bug,
+			Reproducing: repros[bug.Title],
+		})
+	}
+	sort.Slice(list, func(i, j int) bool {
+		first, second := list[i], list[j]
+		firstPatched, secondPatched := first.PatchedOnly(), second.PatchedOnly()
+		if firstPatched != secondPatched {
+			return firstPatched
+		}
+		return first.Title < second.Title
+	})
+	return list
+}
+
+func (serv *HTTPServer) nowReproducing() map[string]bool {
 	if reproLoop := serv.ReproLoop.Load(); reproLoop != nil {
-		repros = reproLoop.Reproducing()
+		return reproLoop.Reproducing()
 	}
+	return nil
+}
+
+func (serv *HTTPServer) collectCrashes(workdir string) ([]*UICrashType, error) {
 	list, err := serv.CrashStore.BugList()
 	if err != nil {
 		return nil, err
 	}
+	repros := serv.nowReproducing()
 	var ret []*UICrashType
 	for _, info := range list {
 		ret = append(ret, makeUICrashType(info, serv.StartTime, repros))
@@ -788,9 +841,17 @@ type UISummaryData struct {
 	Expert       bool
 	Stats        []UIStat
 	Crashes      []*UICrashType
+	PatchedOnly  *UIDiffTable
+	AffectsBoth  *UIDiffTable
+	InProgress   *UIDiffTable
 	Log          string
 }
 
+type UIDiffTable struct {
+	Title string
+	List  []UIDiffBug
+}
+
 type UIVMData struct {
 	Name string
 	VMs  []UIVMInfo
@@ -825,6 +886,11 @@ type UICrash struct {
 	Active bool
 }
 
+type UIDiffBug struct {
+	DiffBug
+	Reproducing bool
+}
+
 type UIStat struct {
 	Name  string
 	Value string
@@ -881,6 +947,7 @@ var summaryTemplate = pages.Create(`
 	{{end}}
 </table>
 
+{{if .Crashes}}
 <table class="list_table">
 	<caption>Crashes:</caption>
 	<tr>
@@ -905,6 +972,63 @@ var summaryTemplate = pages.Create(`
 	</tr>
 	{{end}}
 </table>
+{{end}}
+
+{{define "diff_crashes"}}
+<table class="list_table">
+	<caption>{{.Title}}:</caption>
+	<tr>
+		<th>Description</th>
+		<th>Base</th>
+		<th>Patched</th>
+	</tr>
+	{{range $bug := .List}}
+	<tr>
+		<td class="title">{{$bug.Title}}</td>
+		<td class="title">
+		{{if gt $bug.Base.Crashes 0}}
+			{{$bug.Base.Crashes}} crashes
+		{{else if $bug.Base.NotCrashed}}
+			Not affected
+		{{else}} ? {{end}}
+		{{if $bug.Base.Report}}
+			<a href="/file?name={{$bug.Base.Report}}">[report]</a>
+		{{end}}
+		</td>
+		<td class="title">
+		{{if gt $bug.Patched.Crashes 0}}
+			{{$bug.Patched.Crashes}} crashes
+		{{else}} ? {{end}}
+		{{if $bug.Patched.Report}}
+			<a href="/file?name={{$bug.Patched.Report}}">[report]</a>
+		{{end}}
+		{{if $bug.Patched.CrashLog}}
+			<a href="/file?name={{$bug.Patched.CrashLog}}">[crash log]</a>
+		{{end}}
+		{{if $bug.Patched.Repro}}
+			<a href="/file?name={{$bug.Patched.Repro}}">[syz repro]</a>
+		{{end}}
+		{{if $bug.Patched.ReproLog}}
+			<a href="/file?name={{$bug.Patched.ReproLog}}">[repro log]</a>
+		{{end}}
+		{{if $bug.Reproducing}}[reproducing]{{end}}
+		</td>
+	</tr>
+	{{end}}
+</table>
+{{end}}
+
+{{if .PatchedOnly}}
+{{template "diff_crashes" .PatchedOnly}}
+{{end}}
+
+{{if .AffectsBoth}}
+{{template "diff_crashes" .AffectsBoth}}
+{{end}}
+
+{{if .InProgress}}
+{{template "diff_crashes" .InProgress}}
+{{end}}
 
 <b>Log:</b>
 <br>
diff --git a/pkg/manager/report_generator.go b/pkg/manager/report_generator.go
index b3d293ef1..6dcdbf0a3 100644
--- a/pkg/manager/report_generator.go
+++ b/pkg/manager/report_generator.go
@@ -68,3 +68,12 @@ func CoverToPCs(cfg *mgrconfig.Config, cov []uint64) []uint64 {
 	}
 	return pcs
 }
+
+func PCsToCover(cfg *mgrconfig.Config, pcs map[uint64]struct{}) map[uint64]struct{} {
+	ret := make(map[uint64]struct{})
+	for pc := range pcs {
+		next := backend.NextInstructionPC(cfg.SysTarget, cfg.Type, pc)
+		ret[next] = struct{}{}
+	}
+	return ret
+}
diff --git a/tools/syz-diff/diff.go b/tools/syz-diff/diff.go
new file mode 100644
index 000000000..e593205bc
--- /dev/null
+++ b/tools/syz-diff/diff.go
@@ -0,0 +1,492 @@
+// Copyright 2024 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"math/rand"
+	"net"
+	"os"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/google/syzkaller/pkg/corpus"
+	"github.com/google/syzkaller/pkg/flatrpc"
+	"github.com/google/syzkaller/pkg/fuzzer"
+	"github.com/google/syzkaller/pkg/fuzzer/queue"
+	"github.com/google/syzkaller/pkg/instance"
+	"github.com/google/syzkaller/pkg/log"
+	"github.com/google/syzkaller/pkg/manager"
+	"github.com/google/syzkaller/pkg/mgrconfig"
+	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/pkg/report"
+	"github.com/google/syzkaller/pkg/repro"
+	"github.com/google/syzkaller/pkg/rpcserver"
+	"github.com/google/syzkaller/pkg/signal"
+	"github.com/google/syzkaller/pkg/stat"
+	"github.com/google/syzkaller/pkg/vminfo"
+	"github.com/google/syzkaller/prog"
+	"github.com/google/syzkaller/vm"
+	"github.com/google/syzkaller/vm/dispatcher"
+)
+
+var (
+	flagBaseConfig = flag.String("base", "", "base config")
+	flagNewConfig  = flag.String("new", "", "new config (treated as the main one)")
+	flagDebug      = flag.Bool("debug", false, "dump all VM output to console")
+	flagPatch      = flag.String("patch", "", "a git patch")
+)
+
+func main() {
+	if !prog.GitRevisionKnown() {
+		log.Fatalf("bad syz-diff build: build with make, run bin/syz-diff")
+	}
+	flag.Parse()
+	log.EnableLogCaching(1000, 1<<20)
+
+	baseCfg, err := mgrconfig.LoadFile(*flagBaseConfig)
+	if err != nil {
+		log.Fatalf("base config: %v", err)
+	}
+
+	ctx := vm.ShutdownCtx()
+	base := setup(ctx, "base", baseCfg)
+
+	newCfg, err := mgrconfig.LoadFile(*flagNewConfig)
+	if err != nil {
+		log.Fatalf("new config: %v", err)
+	}
+
+	if *flagPatch != "" {
+		data, err := os.ReadFile(*flagPatch)
+		if err != nil {
+			log.Fatal(err)
+		}
+		extractModifiedFiles(newCfg, data)
+	}
+
+	new := setup(ctx, "new", newCfg)
+	go func() {
+		new.candidates <- manager.LoadSeeds(newCfg, true).Candidates
+	}()
+
+	stream := queue.NewRandomQueue(4096, rand.New(rand.NewSource(time.Now().UnixNano())))
+	base.source = stream
+	new.duplicateInto = stream
+
+	store := &manager.DiffFuzzerStore{BasePath: new.cfg.Workdir}
+	diffCtx := &diffContext{
+		doneRepro:     make(chan *manager.ReproResult),
+		base:          base,
+		new:           new,
+		store:         store,
+		reproAttempts: map[string]int{},
+		http: &manager.HTTPServer{
+			Cfg:       newCfg,
+			StartTime: time.Now(),
+			Corpus:    new.corpus,
+			DiffStore: store,
+		},
+	}
+	diffCtx.http.Pools.Store(new.name, new.pool)
+	diffCtx.http.Pools.Store(base.name, base.pool)
+	new.http = diffCtx.http
+
+	diffCtx.Loop(ctx)
+}
+
+type diffContext struct {
+	store *manager.DiffFuzzerStore
+	http  *manager.HTTPServer
+
+	doneRepro chan *manager.ReproResult
+	base      *kernelContext
+	new       *kernelContext
+
+	mu            sync.Mutex
+	reproAttempts map[string]int
+}
+
+func (dc *diffContext) Loop(ctx context.Context) {
+	reproLoop := manager.NewReproLoop(dc, dc.new.pool.Total()-dc.new.cfg.FuzzingVMs, false)
+	dc.http.ReproLoop.Store(reproLoop)
+	go func() {
+		// Let both base and patched instances somewhat progress in fuzzing before we take
+		// VMs away for bug reproduction.
+		// TODO: determine the exact moment of corpus triage.
+		time.Sleep(15 * time.Minute)
+		log.Logf(0, "starting bug reproductions")
+		reproLoop.Loop(ctx)
+	}()
+
+	go dc.base.Loop()
+	go dc.new.Loop()
+	go dc.http.Serve()
+
+	runner := &reproRunner{done: make(chan reproRunnerResult, 2), kernel: dc.base}
+
+	rareStat := time.NewTicker(5 * time.Minute)
+	for {
+		select {
+		case <-rareStat.C:
+			vals := make(map[string]int)
+			for _, stat := range stat.Collect(stat.All) {
+				vals[stat.Name] = stat.V
+			}
+			data, _ := json.MarshalIndent(vals, "", "  ")
+			log.Logf(0, "STAT %s", data)
+		case rep := <-dc.base.crashes:
+			log.Logf(1, "base crash: %v", rep.Title)
+			dc.store.BaseCrashed(rep.Title, rep.Report)
+		case ret := <-runner.done:
+			if ret.crashTitle == "" {
+				dc.store.BaseNotCrashed(ret.originalTitle)
+				log.Logf(0, "patched-only: %s", ret.originalTitle)
+			} else {
+				dc.store.BaseCrashed(ret.originalTitle, ret.report)
+				log.Logf(0, "crashes both: %s / %s", ret.originalTitle, ret.crashTitle)
+			}
+		case ret := <-dc.doneRepro:
+			if ret.Repro != nil && ret.Repro.Report != nil {
+				origTitle := ret.Crash.Report.Title
+				if ret.Repro.Report.Title == origTitle {
+					origTitle = "-SAME-"
+				}
+				log.Logf(1, "found repro for %q (orig title: %q), took %.2f minutes",
+					ret.Repro.Report.Title, origTitle, ret.Stats.TotalTime.Minutes())
+				go runner.Run(ret.Repro)
+			} else {
+				origTitle := ret.Crash.Report.Title
+				log.Logf(1, "failed repro for %q, err=%s", origTitle, ret.Err)
+			}
+			dc.store.SaveRepro(ret)
+		case rep := <-dc.new.crashes:
+			crash := &manager.Crash{Report: rep}
+			need := dc.NeedRepro(crash)
+			log.Logf(0, "patched crashed: %v [need repro = %v]",
+				rep.Title, need)
+			dc.store.PatchedCrashed(rep.Title, rep.Report, rep.Output)
+			if need {
+				reproLoop.Enqueue(crash)
+			}
+		}
+	}
+}
+
+// TODO: instead of this limit, consider expotentially growing delays between reproduction attempts.
+const maxReproAttempts = 6
+
+func (dc *diffContext) NeedRepro(crash *manager.Crash) bool {
+	if strings.Contains(crash.Title, "no output") ||
+		strings.Contains(crash.Title, "lost connection") ||
+		strings.Contains(crash.Title, "stall") ||
+		strings.Contains(crash.Title, "SYZ") {
+		// Don't waste time reproducing these.
+		return false
+	}
+	dc.mu.Lock()
+	defer dc.mu.Unlock()
+	if dc.store.EverCrashedBase(crash.Title) {
+		return false
+	}
+	if dc.reproAttempts[crash.Title] > maxReproAttempts {
+		return false
+	}
+	return true
+}
+
+func (dc *diffContext) RunRepro(crash *manager.Crash) *manager.ReproResult {
+	dc.mu.Lock()
+	dc.reproAttempts[crash.Title]++
+	dc.mu.Unlock()
+
+	res, stats, err := repro.Run(crash.Output, dc.new.cfg, dc.new.features,
+		dc.new.reporter, dc.new.pool, repro.Fast)
+	if res != nil && res.Report != nil {
+		dc.mu.Lock()
+		dc.reproAttempts[res.Report.Title] = maxReproAttempts
+		dc.mu.Unlock()
+	}
+	ret := &manager.ReproResult{
+		Crash: crash,
+		Repro: res,
+		Stats: stats,
+		Err:   err,
+	}
+	dc.doneRepro <- ret
+	return ret
+}
+
+func (dc *diffContext) ResizeReproPool(size int) {
+	dc.new.pool.ReserveForRun(size)
+}
+
+type kernelContext struct {
+	name       string
+	ctx        context.Context
+	cfg        *mgrconfig.Config
+	reporter   *report.Reporter
+	corpus     *corpus.Corpus
+	fuzzer     atomic.Pointer[fuzzer.Fuzzer]
+	serv       rpcserver.Server
+	servStats  rpcserver.Stats
+	crashes    chan *report.Report
+	pool       *dispatcher.Pool[*vm.Instance]
+	features   flatrpc.Feature
+	candidates chan []fuzzer.Candidate
+
+	coverFilters    manager.CoverageFilters
+	reportGenerator *manager.ReportGeneratorWrapper
+
+	http          *manager.HTTPServer
+	source        queue.Source
+	duplicateInto queue.Executor
+}
+
+func setup(ctx context.Context, name string, cfg *mgrconfig.Config) *kernelContext {
+	osutil.MkdirAll(cfg.Workdir)
+
+	kernelCtx := &kernelContext{
+		name:            name,
+		ctx:             ctx,
+		cfg:             cfg,
+		corpus:          corpus.NewCorpus(ctx),
+		crashes:         make(chan *report.Report, 128),
+		candidates:      make(chan []fuzzer.Candidate),
+		servStats:       rpcserver.NewNamedStats(name),
+		reportGenerator: manager.ReportGeneratorCache(cfg),
+	}
+
+	var err error
+	kernelCtx.reporter, err = report.NewReporter(cfg)
+	if err != nil {
+		log.Fatalf("failed to create reporter for %q: %v", name, err)
+	}
+
+	kernelCtx.serv, err = rpcserver.New(cfg, kernelCtx, kernelCtx.servStats, *flagDebug)
+	if err != nil {
+		log.Fatalf("failed to create rpc server for %q: %v", name, err)
+	}
+
+	vmPool, err := vm.Create(cfg, *flagDebug)
+	if err != nil {
+		log.Fatalf("failed to create vm.Pool for %q: %v", name, err)
+	}
+
+	kernelCtx.pool = vm.NewDispatcher(vmPool, kernelCtx.fuzzerInstance)
+	return kernelCtx
+}
+
+func (kc *kernelContext) Loop() {
+	if err := kc.serv.Listen(); err != nil {
+		log.Fatalf("failed to start rpc server: %v", err)
+	}
+	kc.pool.Loop(kc.ctx)
+}
+
+func (kc *kernelContext) MaxSignal() signal.Signal {
+	if fuzzer := kc.fuzzer.Load(); fuzzer != nil {
+		return fuzzer.Cover.CopyMaxSignal()
+	}
+	return nil
+}
+
+func (kc *kernelContext) BugFrames() (leaks, races []string) {
+	return nil, nil
+}
+
+func (kc *kernelContext) MachineChecked(features flatrpc.Feature, syscalls map[*prog.Syscall]bool) queue.Source {
+	if len(syscalls) == 0 {
+		log.Fatalf("all system calls are disabled")
+	}
+	log.Logf(0, "%s: machine check complete", kc.name)
+	kc.features = features
+
+	var source queue.Source
+	if kc.source == nil {
+		source = queue.Tee(kc.setupFuzzer(features, syscalls), kc.duplicateInto)
+	} else {
+		source = kc.source
+	}
+	opts := fuzzer.DefaultExecOpts(kc.cfg, features, *flagDebug)
+	return queue.DefaultOpts(source, opts)
+}
+
+func (kc *kernelContext) setupFuzzer(features flatrpc.Feature, syscalls map[*prog.Syscall]bool) queue.Source {
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+	fuzzerObj := fuzzer.NewFuzzer(context.Background(), &fuzzer.Config{
+		Corpus:   kc.corpus,
+		Coverage: kc.cfg.Cover,
+		// TODO: it may be unstable between different revisions though.
+		// For now it's only kept true because it seems to increase repro chances in local runs (???).
+		FaultInjection: true,
+		Comparisons:    features&flatrpc.FeatureComparisons != 0,
+		Collide:        true,
+		EnabledCalls:   syscalls,
+		NoMutateCalls:  kc.cfg.NoMutateCalls,
+		PatchTest:      true,
+		Logf: func(level int, msg string, args ...interface{}) {
+			if level != 0 {
+				return
+			}
+			log.Logf(level, msg, args...)
+		},
+	}, rnd, kc.cfg.Target)
+	kc.fuzzer.Store(fuzzerObj)
+
+	if kc.http != nil {
+		kc.http.Fuzzer.Store(fuzzerObj)
+		kc.http.EnabledSyscalls.Store(syscalls)
+	}
+
+	filtered := manager.FilterCandidates(<-kc.candidates, syscalls, false).Candidates
+	log.Logf(0, "%s: adding %d seeds", kc.name, len(filtered))
+	fuzzerObj.AddCandidates(filtered)
+
+	go func() {
+		if !kc.cfg.Cover {
+			return
+		}
+		for {
+			select {
+			case <-time.After(time.Second):
+			case <-kc.ctx.Done():
+				return
+			}
+			newSignal := fuzzerObj.Cover.GrabSignalDelta()
+			if len(newSignal) == 0 {
+				continue
+			}
+			kc.serv.DistributeSignalDelta(newSignal)
+		}
+	}()
+	return fuzzerObj
+}
+
+func (kc *kernelContext) CoverageFilter(modules []*vminfo.KernelModule) []uint64 {
+	kc.reportGenerator.Init(modules)
+	filters, err := manager.PrepareCoverageFilters(kc.reportGenerator, kc.cfg, false)
+	if err != nil {
+		log.Fatalf("failed to init coverage filter: %v", err)
+	}
+	kc.coverFilters = filters
+	kc.corpus.SetFocusAreas(filters.Areas)
+	log.Logf(0, "cover filter size: %d", len(filters.ExecutorFilter))
+	if kc.http != nil {
+		kc.http.Cover.Store(&manager.CoverageInfo{
+			Modules:         modules,
+			ReportGenerator: kc.reportGenerator,
+			CoverFilter:     filters.ExecutorFilter,
+		})
+	}
+	var pcs []uint64
+	for pc := range filters.ExecutorFilter {
+		pcs = append(pcs, pc)
+	}
+	return pcs
+}
+
+func (kc *kernelContext) fuzzerInstance(ctx context.Context, inst *vm.Instance, updInfo dispatcher.UpdateInfo) {
+	index := inst.Index()
+	injectExec := make(chan bool, 10)
+	kc.serv.CreateInstance(index, injectExec, updInfo)
+	rep, err := kc.runInstance(ctx, inst, injectExec)
+	lastExec, _ := kc.serv.ShutdownInstance(index, rep != nil)
+	if rep != nil {
+		rpcserver.PrependExecuting(rep, lastExec)
+		kc.crashes <- rep
+	}
+	if err != nil {
+		log.Errorf("#%d run failed: %s", inst.Index(), err)
+	}
+}
+
+func (kc *kernelContext) runInstance(ctx context.Context, inst *vm.Instance,
+	injectExec <-chan bool) (*report.Report, error) {
+	fwdAddr, err := inst.Forward(kc.serv.Port())
+	if err != nil {
+		return nil, fmt.Errorf("failed to setup port forwarding: %w", err)
+	}
+	executorBin, err := inst.Copy(kc.cfg.ExecutorBin)
+	if err != nil {
+		return nil, fmt.Errorf("failed to copy binary: %w", err)
+	}
+	host, port, err := net.SplitHostPort(fwdAddr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse manager's address")
+	}
+	cmd := fmt.Sprintf("%v runner %v %v %v", executorBin, inst.Index(), host, port)
+	_, rep, err := inst.Run(kc.cfg.Timeouts.VMRunningTime, kc.reporter, cmd,
+		vm.ExitTimeout, vm.StopContext(ctx), vm.InjectExecuting(injectExec),
+		vm.EarlyFinishCb(func() {
+			// Depending on the crash type and kernel config, fuzzing may continue
+			// running for several seconds even after kernel has printed a crash report.
+			// This litters the log and we want to prevent it.
+			kc.serv.StopFuzzing(inst.Index())
+		}),
+	)
+	return rep, err
+}
+
+// reproRunner is used to run reproducers on the base kernel to determine whether it is affected.
+type reproRunner struct {
+	done    chan reproRunnerResult
+	running atomic.Int64
+	kernel  *kernelContext
+}
+
+type reproRunnerResult struct {
+	originalTitle string
+	crashTitle    string
+	report        []byte
+}
+
+func (rr *reproRunner) Run(r *repro.Result) {
+	pool := rr.kernel.pool
+	cnt := int(rr.running.Add(1))
+	pool.ReserveForRun(min(cnt, pool.Total()))
+	defer func() {
+		cnt := int(rr.running.Add(-1))
+		rr.kernel.pool.ReserveForRun(min(cnt, pool.Total()))
+	}()
+
+	ret := reproRunnerResult{originalTitle: r.Report.Title}
+
+	var result *instance.RunResult
+	var err error
+	for i := 0; i < 3; i++ {
+		opts := r.Opts
+		opts.Repeat = true
+		if i == 0 || i == 1 {
+			// Two times out of 3, test with Threaded=true.
+			// The third time we leave it as is in case it was important.
+			opts.Threaded = true
+		}
+		pool.Run(func(ctx context.Context, inst *vm.Instance, updInfo dispatcher.UpdateInfo) {
+			var ret *instance.ExecProgInstance
+			ret, err = instance.SetupExecProg(inst, rr.kernel.cfg, rr.kernel.reporter, nil)
+			if err != nil {
+				return
+			}
+			result, err = ret.RunSyzProg(r.Prog.Serialize(), max(r.Duration, time.Minute), opts,
+				instance.SyzExitConditions)
+		})
+		crashed := result != nil && result.Report != nil
+		log.Logf(1, "attempt #%d to run %q on base: crashed=%v", i, ret.originalTitle, crashed)
+		if crashed {
+			ret.crashTitle = result.Report.Title
+			break
+		}
+	}
+	if err != nil {
+		log.Errorf("failed to run repro: %v", err)
+		return
+	}
+	rr.done <- ret
+}
diff --git a/tools/syz-diff/patch.go b/tools/syz-diff/patch.go
new file mode 100644
index 000000000..6bdcc8e1e
--- /dev/null
+++ b/tools/syz-diff/patch.go
@@ -0,0 +1,94 @@
+// Copyright 2024 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/google/syzkaller/pkg/log"
+	"github.com/google/syzkaller/pkg/mgrconfig"
+	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/pkg/vcs"
+)
+
+func extractModifiedFiles(cfg *mgrconfig.Config, data []byte) {
+	const maxAffectedByHeader = 50
+
+	names := map[string]bool{}
+	includedNames := map[string]bool{}
+	for _, file := range vcs.ParseGitDiff(data) {
+		names[file] = true
+
+		if strings.HasSuffix(file, ".h") && cfg.KernelSrc != "" {
+			// Ideally, we should combine this with the recompilation process - then we know
+			// exactly which files were affected by the patch.
+			out, err := osutil.RunCmd(time.Minute, cfg.KernelSrc, "/usr/bin/grep",
+				"-rl", "--include", `*.c`, `<`+strings.TrimPrefix(file, "include/")+`>`)
+			if err != nil {
+				log.Logf(0, "failed to grep for the header usages: %v", err)
+				continue
+			}
+			lines := strings.Split(string(out), "\n")
+			if len(lines) >= maxAffectedByHeader {
+				// It's too widespread. It won't help us focus on anything.
+				log.Logf(0, "the header %q is included in too many files (%d)", file, len(lines))
+				continue
+			}
+			for _, name := range lines {
+				name = strings.TrimSpace(name)
+				if name == "" {
+					continue
+				}
+				includedNames[name] = true
+			}
+		}
+	}
+
+	var namesList, includedList []string
+	for name := range names {
+		namesList = append(namesList, name)
+	}
+	for name := range includedNames {
+		if names[name] {
+			continue
+		}
+		includedList = append(includedList, name)
+	}
+
+	if len(namesList) > 0 {
+		sort.Strings(namesList)
+		log.Logf(0, "adding the following modified files to focus_order: %q", namesList)
+		cfg.Experimental.FocusAreas = append(cfg.Experimental.FocusAreas,
+			mgrconfig.FocusArea{
+				Name: "modified",
+				Filter: mgrconfig.CovFilterCfg{
+					Files: namesList,
+				},
+				Weight: 3.0,
+			})
+	}
+
+	if len(includedList) > 0 {
+		sort.Strings(includedList)
+		log.Logf(0, "adding the following included files to focus_order: %q", includedList)
+		cfg.Experimental.FocusAreas = append(cfg.Experimental.FocusAreas,
+			mgrconfig.FocusArea{
+				Name: "included",
+				Filter: mgrconfig.CovFilterCfg{
+					Files: includedList,
+				},
+				Weight: 2.0,
+			})
+	}
+
+	// Still fuzz the rest of the kernel.
+	if len(cfg.Experimental.FocusAreas) > 0 {
+		cfg.Experimental.FocusAreas = append(cfg.Experimental.FocusAreas,
+			mgrconfig.FocusArea{
+				Weight: 1.0,
+			})
+	}
+}