pkg/rpctype, syz-runner, syz-verifier: add reruns to syz-verifier architecture

When a mismatch is found in the results returned for a program, the program will be rerun on all the kernels to ensure
the mismatch is not flaky (i.e. it didn't occur because of some background activity or external state and will always
be returned when running the program). If the same mismatch occurs in all reruns, syz-verifier creates a report for
the program, otherwise it discards the program as being flaky

7 files changed, 197 insertions, 137 deletions

diff --git a/syz-verifier/main.go b/syz-verifier/main.go
index 62de2cdf1..eea9d92ed 100755
--- a/syz-verifier/main.go
+++ b/syz-verifier/main.go
@@ -60,18 +60,20 @@ type Verifier struct {
 	stats         *Stats
 	statsWrite    io.Writer
 	newEnv        bool
+	reruns        int
 }
 
 // RPCServer is a wrapper around the rpc.Server. It communicates with  Runners,
 // generates programs and sends complete Results for verification.
 type RPCServer struct {
-	vrf        *Verifier
-	port       int
-	mu         sync.Mutex
-	cond       *sync.Cond
-	pools      map[int]*poolInfo
-	progs      map[int]*progInfo
-	notChecked int
+	vrf             *Verifier
+	port            int
+	mu              sync.Mutex
+	cond            *sync.Cond
+	pools           map[int]*poolInfo
+	progs           map[int]*progInfo
+	notChecked      int
+	rerunsAvailable *sync.Cond
 }
 
 // poolInfo contains kernel-specific information for spawning virtual machines
@@ -82,12 +84,14 @@ type poolInfo struct {
 	cfg      *mgrconfig.Config
 	pool     *vm.Pool
 	Reporter report.Reporter
-	//  vmRunners keeps track of what programs have been sent to each Runner.
+	//  runners keeps track of what programs have been sent to each Runner.
 	//  There is one Runner executing per VM instance.
-	vmRunners map[int]runnerProgs
+	runners map[int]runnerProgs
 	// progs stores the programs that haven't been sent to this kernel yet but
 	// have been sent to at least one other kernel.
 	progs []*progInfo
+	// toRerun stores the programs that still need to be rerun by this kernel.
+	toRerun []*progInfo
 	// checked is set to true when the set of system calls not supported on the
 	// kernel is known.
 	checked bool
@@ -97,9 +101,12 @@ type progInfo struct {
 	prog       *prog.Prog
 	idx        int
 	serialized []byte
-	res        []*Result
+	res        [][]*Result
 	// received stores the number of results received for this program.
 	received int
+
+	runIdx int
+	report *ResultReport
 }
 
 type runnerProgs map[int]*progInfo
@@ -111,6 +118,7 @@ func main() {
 	flagStats := flag.String("stats", "", "where stats will be written when"+
 		"execution of syz-verifier finishes, defaults to stdout")
 	flagEnv := flag.Bool("new-env", true, "create a new environment for each program")
+	flagReruns := flag.Int("rerun", 3, "number of time program is rerun when a mismatch is found")
 	flag.Parse()
 
 	pools := make(map[int]*poolInfo)
@@ -190,7 +198,7 @@ func main() {
 		if err != nil {
 			log.Fatalf("failed to create reporter for instance-%d: %v", idx, err)
 		}
-		pi.vmRunners = make(map[int]runnerProgs)
+		pi.runners = make(map[int]runnerProgs)
 	}
 
 	calls := make(map[*prog.Syscall]bool)
@@ -215,6 +223,7 @@ func main() {
 		reportReasons: len(cfg.EnabledSyscalls) != 0 || len(cfg.DisabledSyscalls) != 0,
 		statsWrite:    sw,
 		newEnv:        *flagEnv,
+		reruns:        *flagReruns,
 	}
 
 	vrf.srv, err = startRPCServer(vrf)
@@ -270,6 +279,7 @@ func startRPCServer(vrf *Verifier) (*RPCServer, error) {
 		notChecked: len(vrf.pools),
 	}
 	srv.cond = sync.NewCond(&srv.mu)
+	srv.rerunsAvailable = sync.NewCond(&srv.mu)
 
 	s, err := rpctype.NewRPCServer(vrf.addr, "Verifier", srv)
 	if err != nil {
@@ -288,7 +298,7 @@ func (srv *RPCServer) Connect(a *rpctype.RunnerConnectArgs, r *rpctype.RunnerCon
 	srv.mu.Lock()
 	defer srv.mu.Unlock()
 	pool, vm := a.Pool, a.VM
-	srv.pools[pool].vmRunners[vm] = make(runnerProgs)
+	srv.pools[pool].runners[vm] = make(runnerProgs)
 	r.CheckUnsupportedCalls = !srv.pools[pool].checked
 	return nil
 }
@@ -362,16 +372,20 @@ func (vrf *Verifier) finalizeCallSet(w io.Writer) {
 func (srv *RPCServer) NextExchange(a *rpctype.NextExchangeArgs, r *rpctype.NextExchangeRes) error {
 	srv.mu.Lock()
 	defer srv.mu.Unlock()
+
+	var res *Result
+	var prog *progInfo
 	if a.Info.Calls != nil {
-		res := &Result{
+		res = &Result{
 			Pool:   a.Pool,
 			Hanged: a.Hanged,
 			Info:   a.Info,
+			RunIdx: a.RunIdx,
 		}
 
-		prog := srv.progs[a.ProgIdx]
+		prog = srv.progs[a.ProgIdx]
 		if prog == nil {
-			// This case can happen if both of the below conditions are true:
+			// This case can happen if both conditions are true:
 			// 1. a Runner calls Verifier.NextExchange, then crashes,
 			// its corresponding Pool being the only one that hasn't
 			// sent results for the program yet
@@ -385,10 +399,11 @@ func (srv *RPCServer) NextExchange(a *rpctype.NextExchangeArgs, r *rpctype.NextE
 			return nil
 		}
 
+		delete(srv.pools[a.Pool].runners[a.VM], prog.idx)
 		if srv.newResult(res, prog) {
-			srv.vrf.processResults(prog.res, prog.prog)
-			delete(srv.progs, a.ProgIdx)
-			delete(srv.pools[a.Pool].vmRunners[a.VM], a.ProgIdx)
+			if srv.vrf.processResults(prog) {
+				delete(srv.progs, prog.idx)
+			}
 		}
 	}
 
@@ -397,8 +412,8 @@ func (srv *RPCServer) NextExchange(a *rpctype.NextExchangeArgs, r *rpctype.NextE
 		srv.cond.Wait()
 	}
 
-	prog, pi := srv.newProgram(a.Pool, a.VM)
-	r.RPCProg = rpctype.RPCProg{Prog: prog, ProgIdx: pi}
+	newProg, pi, ri := srv.newProgram(a.Pool, a.VM)
+	r.RPCProg = rpctype.RPCProg{Prog: newProg, ProgIdx: pi, RunIdx: ri}
 	return nil
 }
 
@@ -406,20 +421,56 @@ func (srv *RPCServer) NextExchange(a *rpctype.NextExchangeArgs, r *rpctype.NextE
 // Results from the corresponding programs have been received and they can be
 // sent for verification. Otherwise, it returns false.
 func (srv *RPCServer) newResult(res *Result, prog *progInfo) bool {
-	prog.res[res.Pool] = res
+	ri := prog.runIdx
+	if prog.res[ri][res.Pool] != nil {
+		return false
+	}
+	prog.res[ri][res.Pool] = res
 	prog.received++
 	return prog.received == len(srv.pools)
 }
 
 // processResults will send a set of complete results for verification and, in
-// case differences are found, it will store a result report highlighting those
-// in th workdir/results directory. If writing the results fails, it returns an
-// error.
-func (vrf *Verifier) processResults(res []*Result, prog *prog.Prog) {
-	vrf.stats.Progs++
-	rr := Verify(res, prog, vrf.stats)
-	if rr == nil {
-		return
+// case differences are found, it will start the rerun process for the program
+// (if reruns are enabled). If every rerun produces the same results, the result
+// report will be printed to persistent storage. Otherwise, the program is
+// discarded as flaky.
+func (vrf *Verifier) processResults(prog *progInfo) bool {
+	// TODO: Simplify this if clause.
+	if prog.runIdx == 0 {
+		vrf.stats.TotalProgs++
+		prog.report = Verify(prog.res[0], prog.prog, vrf.stats)
+		if prog.report == nil {
+			return true
+		}
+	} else {
+		if !VerifyRerun(prog.res[prog.runIdx], prog.report) {
+			vrf.stats.FlakyProgs++
+			log.Printf("flaky results dected: %d", vrf.stats.FlakyProgs)
+			return true
+		}
+	}
+
+	if prog.runIdx < vrf.reruns-1 {
+		vrf.srv.newRun(prog)
+		return false
+	}
+
+	rr := prog.report
+	vrf.stats.MismatchingProgs++
+
+	for _, cr := range rr.Reports {
+		if !cr.Mismatch {
+			break
+		}
+		vrf.stats.Calls[cr.Call].Mismatches++
+		vrf.stats.TotalMismatches++
+		for _, state := range cr.States {
+			if state0 := cr.States[0]; state0 != state {
+				vrf.stats.Calls[cr.Call].States[state] = true
+				vrf.stats.Calls[cr.Call].States[state0] = true
+			}
+		}
 	}
 
 	oldest := 0
@@ -448,6 +499,16 @@ func (vrf *Verifier) processResults(res []*Result, prog *prog.Prog) {
 	}
 
 	log.Printf("result-%d written successfully", oldest)
+	return true
+}
+
+func (srv *RPCServer) newRun(p *progInfo) {
+	p.runIdx++
+	p.received = 0
+	p.res[p.runIdx] = make([]*Result, len(srv.pools))
+	for _, pool := range srv.pools {
+		pool.toRerun = append(pool.toRerun, p)
+	}
 }
 
 func createReport(rr *ResultReport, pools int) []byte {
@@ -476,25 +537,34 @@ func createReport(rr *ResultReport, pools int) []byte {
 
 // newProgram returns a new program for the Runner identified by poolIdx and
 // vmIdx and the program's index.
-func (srv *RPCServer) newProgram(poolIdx, vmIdx int) ([]byte, int) {
+func (srv *RPCServer) newProgram(poolIdx, vmIdx int) ([]byte, int, int) {
 	pool := srv.pools[poolIdx]
+
+	if len(pool.toRerun) != 0 {
+		p := pool.toRerun[0]
+		pool.runners[vmIdx][p.idx] = p
+		pool.toRerun = pool.toRerun[1:]
+		return p.serialized, p.idx, p.runIdx
+	}
+
 	if len(pool.progs) == 0 {
 		prog, progIdx := srv.vrf.generate()
 		pi := &progInfo{
 			prog:       prog,
 			idx:        progIdx,
 			serialized: prog.Serialize(),
-			res:        make([]*Result, len(srv.pools)),
+			res:        make([][]*Result, srv.vrf.reruns),
 		}
+		pi.res[0] = make([]*Result, len(srv.pools))
 		for _, pool := range srv.pools {
 			pool.progs = append(pool.progs, pi)
 		}
 		srv.progs[progIdx] = pi
 	}
 	p := pool.progs[0]
-	pool.vmRunners[vmIdx][p.idx] = p
+	pool.runners[vmIdx][p.idx] = p
 	pool.progs = pool.progs[1:]
-	return p.serialized, p.idx
+	return p.serialized, p.idx, p.runIdx
 }
 
 // generate will return a newly generated program and its index.
@@ -507,13 +577,13 @@ func (vrf *Verifier) generate() (*prog.Prog, int) {
 func (srv *RPCServer) cleanup(poolIdx, vmIdx int) {
 	srv.mu.Lock()
 	defer srv.mu.Unlock()
-	progs := srv.pools[poolIdx].vmRunners[vmIdx]
+	progs := srv.pools[poolIdx].runners[vmIdx]
 
 	for _, prog := range progs {
 		if srv.newResult(&Result{Pool: poolIdx, Crashed: true}, prog) {
-			srv.vrf.processResults(prog.res, prog.prog)
+			srv.vrf.processResults(prog)
 			delete(srv.progs, prog.idx)
-			delete(srv.pools[poolIdx].vmRunners[vmIdx], prog.idx)
+			delete(srv.pools[poolIdx].runners[vmIdx], prog.idx)
 			continue
 		}
 	}
diff --git a/syz-verifier/main_test.go b/syz-verifier/main_test.go
index 0eb502aeb..6bf7c3d7d 100644
--- a/syz-verifier/main_test.go
+++ b/syz-verifier/main_test.go
@@ -40,12 +40,12 @@ func TestNewProgram(t *testing.T) {
 			srv := createTestServer(t)
 			srv.pools = map[int]*poolInfo{
 				1: {
-					vmRunners: map[int]runnerProgs{
+					runners: map[int]runnerProgs{
 						1: {1: {}},
 					},
 					progs: []*progInfo{{idx: 3}},
 				},
-				2: {vmRunners: map[int]runnerProgs{
+				2: {runners: map[int]runnerProgs{
 					2: {1: {}}},
 					progs: []*progInfo{},
 				},
@@ -56,7 +56,7 @@ func TestNewProgram(t *testing.T) {
 				3: {idx: 3},
 			}
 
-			_, gotProgIdx := srv.newProgram(test.pool, test.vm)
+			_, gotProgIdx, _ := srv.newProgram(test.pool, test.vm)
 			if gotProgIdx != test.retProgIdx {
 				t.Errorf("srv.newProgram returned idx: got %d, want %d", gotProgIdx, test.retProgIdx)
 			}
@@ -91,12 +91,17 @@ func TestNewResult(t *testing.T) {
 			srv.pools = map[int]*poolInfo{0: {}, 1: {}}
 			srv.progs = map[int]*progInfo{
 				1: {idx: 1,
-					res: make([]*Result, 2),
+					res: func() [][]*Result {
+						res := make([][]*Result, 1)
+						res[0] = make([]*Result, 2)
+						return res
+					}(),
 				},
 				3: {idx: 3,
-					res: func() []*Result {
-						res := make([]*Result, 2)
-						res[1] = &Result{Pool: 1}
+					res: func() [][]*Result {
+						res := make([][]*Result, 1)
+						res[0] = make([]*Result, 2)
+						res[0][1] = &Result{Pool: 1}
 						return res
 					}(),
 					received: 1,
@@ -114,7 +119,7 @@ func TestConnect(t *testing.T) {
 	srv := createTestServer(t)
 	srv.pools = map[int]*poolInfo{
 		1: {
-			vmRunners: map[int]runnerProgs{
+			runners: map[int]runnerProgs{
 				0: {1: {idx: 1}},
 			},
 			progs: []*progInfo{{
@@ -134,7 +139,7 @@ func TestConnect(t *testing.T) {
 	want, got := map[int]runnerProgs{
 		0: {1: {idx: 1}},
 		1: {},
-	}, srv.pools[a.Pool].vmRunners
+	}, srv.pools[a.Pool].runners
 	if diff := cmp.Diff(want, got, cmp.AllowUnexported(progInfo{})); diff != "" {
 		t.Errorf("srv.progs[a.Name] mismatch (-want +got):\n%s", diff)
 	}
@@ -282,8 +287,8 @@ func TestUpdateUnsupported(t *testing.T) {
 func TestUpdateUnsupportedNotCalledTwice(t *testing.T) {
 	vrf := Verifier{
 		pools: map[int]*poolInfo{
-			0: {vmRunners: map[int]runnerProgs{0: nil, 1: nil}, checked: false},
-			1: {vmRunners: map[int]runnerProgs{}, checked: false},
+			0: {runners: map[int]runnerProgs{0: nil, 1: nil}, checked: false},
+			1: {runners: map[int]runnerProgs{}, checked: false},
 		},
 	}
 	srv, err := startRPCServer(&vrf)
@@ -307,8 +312,8 @@ func TestUpdateUnsupportedNotCalledTwice(t *testing.T) {
 	}
 
 	wantPools := map[int]*poolInfo{
-		0: {vmRunners: map[int]runnerProgs{0: nil, 1: nil}, checked: true},
-		1: {vmRunners: map[int]runnerProgs{}, checked: false},
+		0: {runners: map[int]runnerProgs{0: nil, 1: nil}, checked: true},
+		1: {runners: map[int]runnerProgs{}, checked: false},
 	}
 	if diff := cmp.Diff(wantPools, srv.pools, cmp.AllowUnexported(poolInfo{}, progInfo{})); diff != "" {
 		t.Errorf("srv.pools mismatch (-want +got):\n%s", diff)
@@ -332,7 +337,7 @@ func TestProcessResults(t *testing.T) {
 			wantExist: true,
 			wantStats: &Stats{
 				TotalMismatches: 1,
-				Progs:           1,
+				TotalProgs:      1,
 				Calls: map[string]*CallStats{
 					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[ReturnState]bool{}),
 					"test$res0":      makeCallStats("test$res0", 1, 1, map[ReturnState]bool{{Errno: 2}: true, {Errno: 5}: true}),
@@ -347,7 +352,7 @@ func TestProcessResults(t *testing.T) {
 				makeResult(1, []int{11, 33, 22}),
 			},
 			wantStats: &Stats{
-				Progs: 1,
+				TotalProgs: 1,
 				Calls: map[string]*CallStats{
 					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[ReturnState]bool{}),
 					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[ReturnState]bool{}),
@@ -359,13 +364,20 @@ func TestProcessResults(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			prog := getTestProgram(t)
+			pi := &progInfo{
+				prog: prog,
+				res: func() [][]*Result {
+					res := make([][]*Result, 1)
+					res[0] = test.res
+					return res
+				}()}
 			vrf := Verifier{
 				resultsdir: makeTestResultDirectory(t),
 				stats:      emptyTestStats(),
 			}
 			resultFile := filepath.Join(vrf.resultsdir, "result-0")
 
-			vrf.processResults(test.res, prog)
+			vrf.processResults(pi)
 
 			if diff := cmp.Diff(test.wantStats, vrf.stats); diff != "" {
 				t.Errorf("vrf.stats mismatch (-want +got):\n%s", diff)
@@ -435,15 +447,16 @@ func TestCleanup(t *testing.T) {
 				4: {
 					idx:      4,
 					received: 0,
-					res: func() []*Result {
-						res := make([]*Result, 3)
+					res: func() [][]*Result {
+						res := make([][]*Result, 1)
+						res[0] = make([]*Result, 3)
 						return res
 					}(),
 				}},
 			wantProg: &progInfo{
 				idx:      4,
 				received: 1,
-				res:      []*Result{makeResultCrashed(0), nil, nil},
+				res:      [][]*Result{{makeResultCrashed(0), nil, nil}},
 			},
 			wantStats:  emptyTestStats(),
 			fileExists: false,
@@ -455,15 +468,16 @@ func TestCleanup(t *testing.T) {
 					idx:      4,
 					prog:     prog,
 					received: 2,
-					res: func() []*Result {
-						res := make([]*Result, 3)
-						res[1] = makeResultCrashed(1)
-						res[2] = makeResultCrashed(2)
+					res: func() [][]*Result {
+						res := make([][]*Result, 1)
+						res[0] = make([]*Result, 3)
+						res[0][1] = makeResultCrashed(1)
+						res[0][2] = makeResultCrashed(2)
 						return res
 					}(),
 				}},
 			wantStats: &Stats{
-				Progs: 1,
+				TotalProgs: 1,
 				Calls: map[string]*CallStats{
 					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[ReturnState]bool{}),
 					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[ReturnState]bool{}),
@@ -479,16 +493,18 @@ func TestCleanup(t *testing.T) {
 					idx:      4,
 					prog:     prog,
 					received: 2,
-					res: func() []*Result {
-						res := make([]*Result, 3)
-						res[1] = makeResult(1, []int{11, 33, 44})
-						res[2] = makeResult(2, []int{11, 33, 22})
+					res: func() [][]*Result {
+						res := make([][]*Result, 1)
+						res[0] = make([]*Result, 3)
+						res[0][1] = makeResult(1, []int{11, 33, 44})
+						res[0][2] = makeResult(2, []int{11, 33, 22})
 						return res
 					}(),
 				}},
 			wantStats: &Stats{
-				TotalMismatches: 3,
-				Progs:           1,
+				TotalMismatches:  3,
+				TotalProgs:       1,
+				MismatchingProgs: 1,
 				Calls: map[string]*CallStats{
 					"breaks_returns": makeCallStats("breaks_returns", 1, 1,
 						map[ReturnState]bool{
@@ -513,7 +529,7 @@ func TestCleanup(t *testing.T) {
 			srv := createTestServer(t)
 			srv.progs = test.progs
 			srv.pools = map[int]*poolInfo{
-				0: {vmRunners: map[int]runnerProgs{
+				0: {runners: map[int]runnerProgs{
 					0: {4: srv.progs[4]}},
 				}, 1: {}, 2: {}}
 			resultFile := filepath.Join(srv.vrf.resultsdir, "result-0")
diff --git a/syz-verifier/stats.go b/syz-verifier/stats.go
index 3321de478..804b6ec75 100644
--- a/syz-verifier/stats.go
+++ b/syz-verifier/stats.go
@@ -18,10 +18,12 @@ import (
 // of the verification process.
 type Stats struct {
 	// Calls stores statistics for all supported system calls.
-	Calls           map[string]*CallStats
-	TotalMismatches int
-	Progs           int
-	StartTime       time.Time
+	Calls            map[string]*CallStats
+	TotalMismatches  int
+	TotalProgs       int
+	FlakyProgs       int
+	MismatchingProgs int
+	StartTime        time.Time
 }
 
 // CallStats stores information used to generate statistics for the
@@ -96,7 +98,11 @@ func (s *Stats) ReportGlobalStats(w io.Writer, deltaTime float64) {
 	tc := s.totalCallsExecuted()
 	fmt.Fprintf(w, "total number of mismatches / total number of calls "+
 		"executed: %d / %d (%0.2f %%)\n\n", s.TotalMismatches, tc, getPercentage(s.TotalMismatches, tc))
-	fmt.Fprintf(w, "programs / minute: %0.2f\n\n", float64(s.Progs)/deltaTime)
+	fmt.Fprintf(w, "programs / minute: %0.2f\n\n", float64(s.TotalProgs)/deltaTime)
+	fmt.Fprintf(w, "true mismatching programs: %d / total number of programs: %d (%0.2f %%)\n",
+		s.MismatchingProgs, s.TotalProgs, getPercentage(s.MismatchingProgs, s.TotalProgs))
+	fmt.Fprintf(w, "flaky programs: %d / total number of programs: %d (%0.2f %%)\n\n",
+		s.FlakyProgs, s.TotalProgs, getPercentage(s.FlakyProgs, s.TotalProgs))
 	cs := s.getOrderedStats()
 	for _, c := range cs {
 		fmt.Fprintf(w, "%s\n", s.ReportCallStats(c.Name))
diff --git a/syz-verifier/stats_test.go b/syz-verifier/stats_test.go
index 81dc1ef2e..38f4f4661 100644
--- a/syz-verifier/stats_test.go
+++ b/syz-verifier/stats_test.go
@@ -11,8 +11,10 @@ import (
 
 func dummyStats() *Stats {
 	return &Stats{
-		Progs:           24,
-		TotalMismatches: 10,
+		TotalProgs:       24,
+		TotalMismatches:  10,
+		FlakyProgs:       4,
+		MismatchingProgs: 6,
 		Calls: map[string]*CallStats{
 			"foo": {"foo", 2, 8, map[ReturnState]bool{
 				returnState(1, 7): true,
@@ -69,6 +71,8 @@ func TestReportGlobalStats(t *testing.T) {
 		"total number of mismatches / total number of calls "+
 			"executed: 10 / 20 (50.00 %)\n\n"+
 			"programs / minute: 2.40\n\n"+
+			"true mismatching programs: 6 / total number of programs: 24 (25.00 %)\n"+
+			"flaky programs: 4 / total number of programs: 24 (16.67 %)\n\n"+
 			"statistics for bar:\n"+
 			"\t↳ mismatches of bar / occurrences of bar: 5 / 6 (83.33 %)\n"+
 			"\t↳ mismatches of bar / total number of mismatches: 5 / 10 (50.00 %)\n"+
diff --git a/syz-verifier/test_utils.go b/syz-verifier/test_utils.go
index 7c5df0cf5..a32c0aa29 100644
--- a/syz-verifier/test_utils.go
+++ b/syz-verifier/test_utils.go
@@ -24,6 +24,7 @@ func createTestServer(t *testing.T) *RPCServer {
 		choiceTable: target.DefaultChoiceTable(),
 		rnd:         rand.New(rand.NewSource(time.Now().UnixNano())),
 		progIdx:     3,
+		reruns:      1,
 	}
 	vrf.resultsdir = makeTestResultDirectory(t)
 	vrf.stats = emptyTestStats()
diff --git a/syz-verifier/verifier.go b/syz-verifier/verifier.go
index c36d9236c..9e97046b9 100644
--- a/syz-verifier/verifier.go
+++ b/syz-verifier/verifier.go
@@ -22,6 +22,8 @@ type Result struct {
 	Info ipc.ProgInfo
 	// Crashed is set to true if a crash occurred while executing the program.
 	Crashed bool
+
+	RunIdx int
 }
 
 type ResultReport struct {
@@ -67,9 +69,29 @@ func (s ReturnState) String() string {
 	return state
 }
 
+// VeifyRerun compares the results obtained from rerunning a program with what
+// was reported in the initial result report.
+func VerifyRerun(res []*Result, rr *ResultReport) bool {
+	for idx, cr := range rr.Reports {
+		for _, r := range res {
+			var state ReturnState
+			if r.Crashed {
+				state = ReturnState{Crashed: true}
+			} else {
+				ci := r.Info.Calls[idx]
+				state = ReturnState{Errno: ci.Errno, Flags: ci.Flags}
+			}
+			if state != cr.States[r.Pool] {
+				return false
+			}
+		}
+	}
+	return true
+}
+
 // Verify checks whether the Results of the same program, executed on different
-// kernels are the same. If that's not the case, it returns a ResultReport
-// which highlights the differences.
+// kernels, are the same. If that's not the case, it returns a ResultReport,
+// highlighting the differences.
 func Verify(res []*Result, prog *prog.Prog, s *Stats) *ResultReport {
 	rr := &ResultReport{
 		Prog: string(prog.Serialize()),
@@ -100,26 +122,14 @@ func Verify(res []*Result, prog *prog.Prog, s *Stats) *ResultReport {
 	var send bool
 	pool0 := res[0].Pool
 	for _, cr := range rr.Reports {
-		cs := s.Calls[cr.Call]
-
-		mismatch := false
 		for _, state := range cr.States {
 			// For each CallReport, verify whether the ReturnStates from all
 			// the pools that executed the program are the same
 			if state0 := cr.States[pool0]; state0 != state {
 				cr.Mismatch = true
 				send = true
-				mismatch = true
-
-				cs.States[state] = true
-				cs.States[state0] = true
 			}
 		}
-
-		if mismatch {
-			cs.Mismatches++
-			s.TotalMismatches++
-		}
 	}
 
 	if send {
diff --git a/syz-verifier/verifier_test.go b/syz-verifier/verifier_test.go
index babd87476..41344bafc 100644
--- a/syz-verifier/verifier_test.go
+++ b/syz-verifier/verifier_test.go
@@ -18,7 +18,7 @@ func TestVerify(t *testing.T) {
 		name       string
 		res        []*Result
 		wantReport *ResultReport
-		wantStats  *Stats
+		wantStats  []*CallStats
 	}{
 		{
 			name: "only crashes",
@@ -27,13 +27,6 @@ func TestVerify(t *testing.T) {
 				makeResultCrashed(4),
 			},
 			wantReport: nil,
-			wantStats: &Stats{
-				Calls: map[string]*CallStats{
-					"breaks_returns": {Name: "breaks_returns", Occurrences: 1, States: map[ReturnState]bool{}},
-					"minimize$0":     {Name: "minimize$0", Occurrences: 1, States: map[ReturnState]bool{}},
-					"test$res0":      {Name: "test$res0", Occurrences: 1, States: map[ReturnState]bool{}},
-				},
-			},
 		},
 		{
 			name: "mismatches because results and crashes",
@@ -62,24 +55,6 @@ func TestVerify(t *testing.T) {
 						Mismatch: true},
 				},
 			},
-			wantStats: &Stats{
-				TotalMismatches: 6,
-				Calls: map[string]*CallStats{
-					"breaks_returns": {Name: "breaks_returns", Occurrences: 1, Mismatches: 2, States: map[ReturnState]bool{
-						crashedReturnState(): true,
-						returnState(11, 1):   true,
-					}},
-					"minimize$0": {Name: "minimize$0", Occurrences: 1,
-						Mismatches: 2, States: map[ReturnState]bool{
-							crashedReturnState(): true,
-							returnState(33, 3):   true,
-						}},
-					"test$res0": {Name: "test$res0", Occurrences: 1,
-						Mismatches: 2, States: map[ReturnState]bool{
-							crashedReturnState(): true,
-							returnState(22, 3):   true}},
-				},
-			},
 		},
 		{
 			name: "mismatches not found in results",
@@ -87,13 +62,6 @@ func TestVerify(t *testing.T) {
 				makeResult(2, []int{11, 33, 22}, []int{1, 3, 3}...),
 				makeResult(4, []int{11, 33, 22}, []int{1, 3, 3}...)},
 			wantReport: nil,
-			wantStats: &Stats{
-				Calls: map[string]*CallStats{
-					"breaks_returns": {Name: "breaks_returns", Occurrences: 1, States: map[ReturnState]bool{}},
-					"minimize$0":     {Name: "minimize$0", Occurrences: 1, States: map[ReturnState]bool{}},
-					"test$res0":      {Name: "test$res0", Occurrences: 1, States: map[ReturnState]bool{}},
-				},
-			},
 		},
 		{
 			name: "mismatches found in results",
@@ -109,19 +77,7 @@ func TestVerify(t *testing.T) {
 					{Call: "test$res0", States: map[int]ReturnState{1: {Errno: 2, Flags: 7}, 4: {Errno: 5, Flags: 3}}, Mismatch: true},
 				},
 			},
-			wantStats: &Stats{
-				TotalMismatches: 1,
-				Calls: map[string]*CallStats{
-					"breaks_returns": {Name: "breaks_returns", Occurrences: 1, States: map[ReturnState]bool{}},
-					"minimize$0":     {Name: "minimize$0", Occurrences: 1, States: map[ReturnState]bool{}},
-					"test$res0": {Name: "test$res0", Occurrences: 1,
-						Mismatches: 1, States: map[ReturnState]bool{
-							{Errno: 2, Flags: 7}: true,
-							{Errno: 5, Flags: 3}: true}},
-				},
-			},
-		},
-	}
+		}}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
@@ -135,9 +91,6 @@ func TestVerify(t *testing.T) {
 			if diff := cmp.Diff(test.wantReport, got); diff != "" {
 				t.Errorf("Verify report mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(test.wantStats, stats); diff != "" {
-				t.Errorf("Verify stats mismatch (-want +got):\n%s", diff)
-			}
 		})
 	}
 }