From f9e341e30b4f3faa468a0b885775a4fbf7825016 Mon Sep 17 00:00:00 2001 From: Mara Mihali Date: Thu, 5 Aug 2021 16:56:46 +0000 Subject: pkg/rpctype, syz-runner, syz-verifier: add reruns to syz-verifier architecture When a mismatch is found in the results returned for a program, the program will be rerun on all the kernels to ensure the mismatch is not flaky (i.e. it didn't occur because of some background activity or external state and will always be returned when running the program). If the same mismatch occurs in all reruns, syz-verifier creates a report for the program, otherwise it discards the program as being flaky --- pkg/rpctype/rpctype.go | 3 + syz-runner/runner.go | 12 ++-- syz-verifier/main.go | 142 +++++++++++++++++++++++++++++++----------- syz-verifier/main_test.go | 78 ++++++++++++++--------- syz-verifier/stats.go | 16 +++-- syz-verifier/stats_test.go | 8 ++- syz-verifier/test_utils.go | 1 + syz-verifier/verifier.go | 38 ++++++----- syz-verifier/verifier_test.go | 51 +-------------- 9 files changed, 206 insertions(+), 143 deletions(-) diff --git a/pkg/rpctype/rpctype.go b/pkg/rpctype/rpctype.go index b64fb1561..35aa75307 100644 --- a/pkg/rpctype/rpctype.go +++ b/pkg/rpctype/rpctype.go @@ -27,6 +27,7 @@ type RPCCandidate struct { type RPCProg struct { Prog []byte ProgIdx int + RunIdx int } type ConnectArgs struct { @@ -113,6 +114,8 @@ type NextExchangeArgs struct { // Info contains information about the execution of each system call in the // program. Info ipc.ProgInfo + // RunIdx is the number of times this program has been run on the kernel. + RunIdx int } // NextExchaneRes contains the data passed from server to client namely diff --git a/syz-runner/runner.go b/syz-runner/runner.go index 033c01525..a01cb794b 100644 --- a/syz-runner/runner.go +++ b/syz-runner/runner.go @@ -33,7 +33,6 @@ func main() { flagOS := flag.String("os", runtime.GOOS, "target OS") flagArch := flag.String("arch", runtime.GOARCH, "target arch") flagEnv := flag.Bool("new-env", true, "create a new environment for each program") - flag.Parse() target, err := prog.GetTarget(*flagOS, *flagArch) @@ -94,13 +93,13 @@ func main() { log.Fatalf("failed to get initial program: %v", err) } - rn.Run(res.Prog, res.ProgIdx) + rn.Run(res.Prog, res.ProgIdx, res.RunIdx) } // Run is responsible for requesting new programs from the verifier, executing them and then sending back the Result. // TODO: Implement functionality to execute several programs at once and send back a slice of results. -func (rn *Runner) Run(firstProg []byte, idx int) { - p, pIdx := firstProg, idx +func (rn *Runner) Run(firstProg []byte, idx, runIdx int) { + p, pIdx, rIdx := firstProg, idx, runIdx env, err := ipc.MakeEnv(rn.config, 0) if err != nil { @@ -124,13 +123,14 @@ func (rn *Runner) Run(firstProg []byte, idx int) { ProgIdx: pIdx, Hanged: hanged, Info: *info, + RunIdx: rIdx, } + r := &rpctype.NextExchangeRes{} if err := rn.vrf.Call("Verifier.NextExchange", a, r); err != nil { log.Fatalf("failed to make exchange with verifier: %v", err) } - p = r.Prog - pIdx = r.ProgIdx + p, pIdx, rIdx = r.Prog, r.ProgIdx, r.RunIdx if !rn.newEnv { continue diff --git a/syz-verifier/main.go b/syz-verifier/main.go index 62de2cdf1..eea9d92ed 100755 --- a/syz-verifier/main.go +++ b/syz-verifier/main.go @@ -60,18 +60,20 @@ type Verifier struct { stats *Stats statsWrite io.Writer newEnv bool + reruns int } // RPCServer is a wrapper around the rpc.Server. It communicates with Runners, // generates programs and sends complete Results for verification. type RPCServer struct { - vrf *Verifier - port int - mu sync.Mutex - cond *sync.Cond - pools map[int]*poolInfo - progs map[int]*progInfo - notChecked int + vrf *Verifier + port int + mu sync.Mutex + cond *sync.Cond + pools map[int]*poolInfo + progs map[int]*progInfo + notChecked int + rerunsAvailable *sync.Cond } // poolInfo contains kernel-specific information for spawning virtual machines @@ -82,12 +84,14 @@ type poolInfo struct { cfg *mgrconfig.Config pool *vm.Pool Reporter report.Reporter - // vmRunners keeps track of what programs have been sent to each Runner. + // runners keeps track of what programs have been sent to each Runner. // There is one Runner executing per VM instance. - vmRunners map[int]runnerProgs + runners map[int]runnerProgs // progs stores the programs that haven't been sent to this kernel yet but // have been sent to at least one other kernel. progs []*progInfo + // toRerun stores the programs that still need to be rerun by this kernel. + toRerun []*progInfo // checked is set to true when the set of system calls not supported on the // kernel is known. checked bool @@ -97,9 +101,12 @@ type progInfo struct { prog *prog.Prog idx int serialized []byte - res []*Result + res [][]*Result // received stores the number of results received for this program. received int + + runIdx int + report *ResultReport } type runnerProgs map[int]*progInfo @@ -111,6 +118,7 @@ func main() { flagStats := flag.String("stats", "", "where stats will be written when"+ "execution of syz-verifier finishes, defaults to stdout") flagEnv := flag.Bool("new-env", true, "create a new environment for each program") + flagReruns := flag.Int("rerun", 3, "number of time program is rerun when a mismatch is found") flag.Parse() pools := make(map[int]*poolInfo) @@ -190,7 +198,7 @@ func main() { if err != nil { log.Fatalf("failed to create reporter for instance-%d: %v", idx, err) } - pi.vmRunners = make(map[int]runnerProgs) + pi.runners = make(map[int]runnerProgs) } calls := make(map[*prog.Syscall]bool) @@ -215,6 +223,7 @@ func main() { reportReasons: len(cfg.EnabledSyscalls) != 0 || len(cfg.DisabledSyscalls) != 0, statsWrite: sw, newEnv: *flagEnv, + reruns: *flagReruns, } vrf.srv, err = startRPCServer(vrf) @@ -270,6 +279,7 @@ func startRPCServer(vrf *Verifier) (*RPCServer, error) { notChecked: len(vrf.pools), } srv.cond = sync.NewCond(&srv.mu) + srv.rerunsAvailable = sync.NewCond(&srv.mu) s, err := rpctype.NewRPCServer(vrf.addr, "Verifier", srv) if err != nil { @@ -288,7 +298,7 @@ func (srv *RPCServer) Connect(a *rpctype.RunnerConnectArgs, r *rpctype.RunnerCon srv.mu.Lock() defer srv.mu.Unlock() pool, vm := a.Pool, a.VM - srv.pools[pool].vmRunners[vm] = make(runnerProgs) + srv.pools[pool].runners[vm] = make(runnerProgs) r.CheckUnsupportedCalls = !srv.pools[pool].checked return nil } @@ -362,16 +372,20 @@ func (vrf *Verifier) finalizeCallSet(w io.Writer) { func (srv *RPCServer) NextExchange(a *rpctype.NextExchangeArgs, r *rpctype.NextExchangeRes) error { srv.mu.Lock() defer srv.mu.Unlock() + + var res *Result + var prog *progInfo if a.Info.Calls != nil { - res := &Result{ + res = &Result{ Pool: a.Pool, Hanged: a.Hanged, Info: a.Info, + RunIdx: a.RunIdx, } - prog := srv.progs[a.ProgIdx] + prog = srv.progs[a.ProgIdx] if prog == nil { - // This case can happen if both of the below conditions are true: + // This case can happen if both conditions are true: // 1. a Runner calls Verifier.NextExchange, then crashes, // its corresponding Pool being the only one that hasn't // sent results for the program yet @@ -385,10 +399,11 @@ func (srv *RPCServer) NextExchange(a *rpctype.NextExchangeArgs, r *rpctype.NextE return nil } + delete(srv.pools[a.Pool].runners[a.VM], prog.idx) if srv.newResult(res, prog) { - srv.vrf.processResults(prog.res, prog.prog) - delete(srv.progs, a.ProgIdx) - delete(srv.pools[a.Pool].vmRunners[a.VM], a.ProgIdx) + if srv.vrf.processResults(prog) { + delete(srv.progs, prog.idx) + } } } @@ -397,8 +412,8 @@ func (srv *RPCServer) NextExchange(a *rpctype.NextExchangeArgs, r *rpctype.NextE srv.cond.Wait() } - prog, pi := srv.newProgram(a.Pool, a.VM) - r.RPCProg = rpctype.RPCProg{Prog: prog, ProgIdx: pi} + newProg, pi, ri := srv.newProgram(a.Pool, a.VM) + r.RPCProg = rpctype.RPCProg{Prog: newProg, ProgIdx: pi, RunIdx: ri} return nil } @@ -406,20 +421,56 @@ func (srv *RPCServer) NextExchange(a *rpctype.NextExchangeArgs, r *rpctype.NextE // Results from the corresponding programs have been received and they can be // sent for verification. Otherwise, it returns false. func (srv *RPCServer) newResult(res *Result, prog *progInfo) bool { - prog.res[res.Pool] = res + ri := prog.runIdx + if prog.res[ri][res.Pool] != nil { + return false + } + prog.res[ri][res.Pool] = res prog.received++ return prog.received == len(srv.pools) } // processResults will send a set of complete results for verification and, in -// case differences are found, it will store a result report highlighting those -// in th workdir/results directory. If writing the results fails, it returns an -// error. -func (vrf *Verifier) processResults(res []*Result, prog *prog.Prog) { - vrf.stats.Progs++ - rr := Verify(res, prog, vrf.stats) - if rr == nil { - return +// case differences are found, it will start the rerun process for the program +// (if reruns are enabled). If every rerun produces the same results, the result +// report will be printed to persistent storage. Otherwise, the program is +// discarded as flaky. +func (vrf *Verifier) processResults(prog *progInfo) bool { + // TODO: Simplify this if clause. + if prog.runIdx == 0 { + vrf.stats.TotalProgs++ + prog.report = Verify(prog.res[0], prog.prog, vrf.stats) + if prog.report == nil { + return true + } + } else { + if !VerifyRerun(prog.res[prog.runIdx], prog.report) { + vrf.stats.FlakyProgs++ + log.Printf("flaky results dected: %d", vrf.stats.FlakyProgs) + return true + } + } + + if prog.runIdx < vrf.reruns-1 { + vrf.srv.newRun(prog) + return false + } + + rr := prog.report + vrf.stats.MismatchingProgs++ + + for _, cr := range rr.Reports { + if !cr.Mismatch { + break + } + vrf.stats.Calls[cr.Call].Mismatches++ + vrf.stats.TotalMismatches++ + for _, state := range cr.States { + if state0 := cr.States[0]; state0 != state { + vrf.stats.Calls[cr.Call].States[state] = true + vrf.stats.Calls[cr.Call].States[state0] = true + } + } } oldest := 0 @@ -448,6 +499,16 @@ func (vrf *Verifier) processResults(res []*Result, prog *prog.Prog) { } log.Printf("result-%d written successfully", oldest) + return true +} + +func (srv *RPCServer) newRun(p *progInfo) { + p.runIdx++ + p.received = 0 + p.res[p.runIdx] = make([]*Result, len(srv.pools)) + for _, pool := range srv.pools { + pool.toRerun = append(pool.toRerun, p) + } } func createReport(rr *ResultReport, pools int) []byte { @@ -476,25 +537,34 @@ func createReport(rr *ResultReport, pools int) []byte { // newProgram returns a new program for the Runner identified by poolIdx and // vmIdx and the program's index. -func (srv *RPCServer) newProgram(poolIdx, vmIdx int) ([]byte, int) { +func (srv *RPCServer) newProgram(poolIdx, vmIdx int) ([]byte, int, int) { pool := srv.pools[poolIdx] + + if len(pool.toRerun) != 0 { + p := pool.toRerun[0] + pool.runners[vmIdx][p.idx] = p + pool.toRerun = pool.toRerun[1:] + return p.serialized, p.idx, p.runIdx + } + if len(pool.progs) == 0 { prog, progIdx := srv.vrf.generate() pi := &progInfo{ prog: prog, idx: progIdx, serialized: prog.Serialize(), - res: make([]*Result, len(srv.pools)), + res: make([][]*Result, srv.vrf.reruns), } + pi.res[0] = make([]*Result, len(srv.pools)) for _, pool := range srv.pools { pool.progs = append(pool.progs, pi) } srv.progs[progIdx] = pi } p := pool.progs[0] - pool.vmRunners[vmIdx][p.idx] = p + pool.runners[vmIdx][p.idx] = p pool.progs = pool.progs[1:] - return p.serialized, p.idx + return p.serialized, p.idx, p.runIdx } // generate will return a newly generated program and its index. @@ -507,13 +577,13 @@ func (vrf *Verifier) generate() (*prog.Prog, int) { func (srv *RPCServer) cleanup(poolIdx, vmIdx int) { srv.mu.Lock() defer srv.mu.Unlock() - progs := srv.pools[poolIdx].vmRunners[vmIdx] + progs := srv.pools[poolIdx].runners[vmIdx] for _, prog := range progs { if srv.newResult(&Result{Pool: poolIdx, Crashed: true}, prog) { - srv.vrf.processResults(prog.res, prog.prog) + srv.vrf.processResults(prog) delete(srv.progs, prog.idx) - delete(srv.pools[poolIdx].vmRunners[vmIdx], prog.idx) + delete(srv.pools[poolIdx].runners[vmIdx], prog.idx) continue } } diff --git a/syz-verifier/main_test.go b/syz-verifier/main_test.go index 0eb502aeb..6bf7c3d7d 100644 --- a/syz-verifier/main_test.go +++ b/syz-verifier/main_test.go @@ -40,12 +40,12 @@ func TestNewProgram(t *testing.T) { srv := createTestServer(t) srv.pools = map[int]*poolInfo{ 1: { - vmRunners: map[int]runnerProgs{ + runners: map[int]runnerProgs{ 1: {1: {}}, }, progs: []*progInfo{{idx: 3}}, }, - 2: {vmRunners: map[int]runnerProgs{ + 2: {runners: map[int]runnerProgs{ 2: {1: {}}}, progs: []*progInfo{}, }, @@ -56,7 +56,7 @@ func TestNewProgram(t *testing.T) { 3: {idx: 3}, } - _, gotProgIdx := srv.newProgram(test.pool, test.vm) + _, gotProgIdx, _ := srv.newProgram(test.pool, test.vm) if gotProgIdx != test.retProgIdx { t.Errorf("srv.newProgram returned idx: got %d, want %d", gotProgIdx, test.retProgIdx) } @@ -91,12 +91,17 @@ func TestNewResult(t *testing.T) { srv.pools = map[int]*poolInfo{0: {}, 1: {}} srv.progs = map[int]*progInfo{ 1: {idx: 1, - res: make([]*Result, 2), + res: func() [][]*Result { + res := make([][]*Result, 1) + res[0] = make([]*Result, 2) + return res + }(), }, 3: {idx: 3, - res: func() []*Result { - res := make([]*Result, 2) - res[1] = &Result{Pool: 1} + res: func() [][]*Result { + res := make([][]*Result, 1) + res[0] = make([]*Result, 2) + res[0][1] = &Result{Pool: 1} return res }(), received: 1, @@ -114,7 +119,7 @@ func TestConnect(t *testing.T) { srv := createTestServer(t) srv.pools = map[int]*poolInfo{ 1: { - vmRunners: map[int]runnerProgs{ + runners: map[int]runnerProgs{ 0: {1: {idx: 1}}, }, progs: []*progInfo{{ @@ -134,7 +139,7 @@ func TestConnect(t *testing.T) { want, got := map[int]runnerProgs{ 0: {1: {idx: 1}}, 1: {}, - }, srv.pools[a.Pool].vmRunners + }, srv.pools[a.Pool].runners if diff := cmp.Diff(want, got, cmp.AllowUnexported(progInfo{})); diff != "" { t.Errorf("srv.progs[a.Name] mismatch (-want +got):\n%s", diff) } @@ -282,8 +287,8 @@ func TestUpdateUnsupported(t *testing.T) { func TestUpdateUnsupportedNotCalledTwice(t *testing.T) { vrf := Verifier{ pools: map[int]*poolInfo{ - 0: {vmRunners: map[int]runnerProgs{0: nil, 1: nil}, checked: false}, - 1: {vmRunners: map[int]runnerProgs{}, checked: false}, + 0: {runners: map[int]runnerProgs{0: nil, 1: nil}, checked: false}, + 1: {runners: map[int]runnerProgs{}, checked: false}, }, } srv, err := startRPCServer(&vrf) @@ -307,8 +312,8 @@ func TestUpdateUnsupportedNotCalledTwice(t *testing.T) { } wantPools := map[int]*poolInfo{ - 0: {vmRunners: map[int]runnerProgs{0: nil, 1: nil}, checked: true}, - 1: {vmRunners: map[int]runnerProgs{}, checked: false}, + 0: {runners: map[int]runnerProgs{0: nil, 1: nil}, checked: true}, + 1: {runners: map[int]runnerProgs{}, checked: false}, } if diff := cmp.Diff(wantPools, srv.pools, cmp.AllowUnexported(poolInfo{}, progInfo{})); diff != "" { t.Errorf("srv.pools mismatch (-want +got):\n%s", diff) @@ -332,7 +337,7 @@ func TestProcessResults(t *testing.T) { wantExist: true, wantStats: &Stats{ TotalMismatches: 1, - Progs: 1, + TotalProgs: 1, Calls: map[string]*CallStats{ "breaks_returns": makeCallStats("breaks_returns", 1, 0, map[ReturnState]bool{}), "test$res0": makeCallStats("test$res0", 1, 1, map[ReturnState]bool{{Errno: 2}: true, {Errno: 5}: true}), @@ -347,7 +352,7 @@ func TestProcessResults(t *testing.T) { makeResult(1, []int{11, 33, 22}), }, wantStats: &Stats{ - Progs: 1, + TotalProgs: 1, Calls: map[string]*CallStats{ "breaks_returns": makeCallStats("breaks_returns", 1, 0, map[ReturnState]bool{}), "minimize$0": makeCallStats("minimize$0", 1, 0, map[ReturnState]bool{}), @@ -359,13 +364,20 @@ func TestProcessResults(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { prog := getTestProgram(t) + pi := &progInfo{ + prog: prog, + res: func() [][]*Result { + res := make([][]*Result, 1) + res[0] = test.res + return res + }()} vrf := Verifier{ resultsdir: makeTestResultDirectory(t), stats: emptyTestStats(), } resultFile := filepath.Join(vrf.resultsdir, "result-0") - vrf.processResults(test.res, prog) + vrf.processResults(pi) if diff := cmp.Diff(test.wantStats, vrf.stats); diff != "" { t.Errorf("vrf.stats mismatch (-want +got):\n%s", diff) @@ -435,15 +447,16 @@ func TestCleanup(t *testing.T) { 4: { idx: 4, received: 0, - res: func() []*Result { - res := make([]*Result, 3) + res: func() [][]*Result { + res := make([][]*Result, 1) + res[0] = make([]*Result, 3) return res }(), }}, wantProg: &progInfo{ idx: 4, received: 1, - res: []*Result{makeResultCrashed(0), nil, nil}, + res: [][]*Result{{makeResultCrashed(0), nil, nil}}, }, wantStats: emptyTestStats(), fileExists: false, @@ -455,15 +468,16 @@ func TestCleanup(t *testing.T) { idx: 4, prog: prog, received: 2, - res: func() []*Result { - res := make([]*Result, 3) - res[1] = makeResultCrashed(1) - res[2] = makeResultCrashed(2) + res: func() [][]*Result { + res := make([][]*Result, 1) + res[0] = make([]*Result, 3) + res[0][1] = makeResultCrashed(1) + res[0][2] = makeResultCrashed(2) return res }(), }}, wantStats: &Stats{ - Progs: 1, + TotalProgs: 1, Calls: map[string]*CallStats{ "breaks_returns": makeCallStats("breaks_returns", 1, 0, map[ReturnState]bool{}), "minimize$0": makeCallStats("minimize$0", 1, 0, map[ReturnState]bool{}), @@ -479,16 +493,18 @@ func TestCleanup(t *testing.T) { idx: 4, prog: prog, received: 2, - res: func() []*Result { - res := make([]*Result, 3) - res[1] = makeResult(1, []int{11, 33, 44}) - res[2] = makeResult(2, []int{11, 33, 22}) + res: func() [][]*Result { + res := make([][]*Result, 1) + res[0] = make([]*Result, 3) + res[0][1] = makeResult(1, []int{11, 33, 44}) + res[0][2] = makeResult(2, []int{11, 33, 22}) return res }(), }}, wantStats: &Stats{ - TotalMismatches: 3, - Progs: 1, + TotalMismatches: 3, + TotalProgs: 1, + MismatchingProgs: 1, Calls: map[string]*CallStats{ "breaks_returns": makeCallStats("breaks_returns", 1, 1, map[ReturnState]bool{ @@ -513,7 +529,7 @@ func TestCleanup(t *testing.T) { srv := createTestServer(t) srv.progs = test.progs srv.pools = map[int]*poolInfo{ - 0: {vmRunners: map[int]runnerProgs{ + 0: {runners: map[int]runnerProgs{ 0: {4: srv.progs[4]}}, }, 1: {}, 2: {}} resultFile := filepath.Join(srv.vrf.resultsdir, "result-0") diff --git a/syz-verifier/stats.go b/syz-verifier/stats.go index 3321de478..804b6ec75 100644 --- a/syz-verifier/stats.go +++ b/syz-verifier/stats.go @@ -18,10 +18,12 @@ import ( // of the verification process. type Stats struct { // Calls stores statistics for all supported system calls. - Calls map[string]*CallStats - TotalMismatches int - Progs int - StartTime time.Time + Calls map[string]*CallStats + TotalMismatches int + TotalProgs int + FlakyProgs int + MismatchingProgs int + StartTime time.Time } // CallStats stores information used to generate statistics for the @@ -96,7 +98,11 @@ func (s *Stats) ReportGlobalStats(w io.Writer, deltaTime float64) { tc := s.totalCallsExecuted() fmt.Fprintf(w, "total number of mismatches / total number of calls "+ "executed: %d / %d (%0.2f %%)\n\n", s.TotalMismatches, tc, getPercentage(s.TotalMismatches, tc)) - fmt.Fprintf(w, "programs / minute: %0.2f\n\n", float64(s.Progs)/deltaTime) + fmt.Fprintf(w, "programs / minute: %0.2f\n\n", float64(s.TotalProgs)/deltaTime) + fmt.Fprintf(w, "true mismatching programs: %d / total number of programs: %d (%0.2f %%)\n", + s.MismatchingProgs, s.TotalProgs, getPercentage(s.MismatchingProgs, s.TotalProgs)) + fmt.Fprintf(w, "flaky programs: %d / total number of programs: %d (%0.2f %%)\n\n", + s.FlakyProgs, s.TotalProgs, getPercentage(s.FlakyProgs, s.TotalProgs)) cs := s.getOrderedStats() for _, c := range cs { fmt.Fprintf(w, "%s\n", s.ReportCallStats(c.Name)) diff --git a/syz-verifier/stats_test.go b/syz-verifier/stats_test.go index 81dc1ef2e..38f4f4661 100644 --- a/syz-verifier/stats_test.go +++ b/syz-verifier/stats_test.go @@ -11,8 +11,10 @@ import ( func dummyStats() *Stats { return &Stats{ - Progs: 24, - TotalMismatches: 10, + TotalProgs: 24, + TotalMismatches: 10, + FlakyProgs: 4, + MismatchingProgs: 6, Calls: map[string]*CallStats{ "foo": {"foo", 2, 8, map[ReturnState]bool{ returnState(1, 7): true, @@ -69,6 +71,8 @@ func TestReportGlobalStats(t *testing.T) { "total number of mismatches / total number of calls "+ "executed: 10 / 20 (50.00 %)\n\n"+ "programs / minute: 2.40\n\n"+ + "true mismatching programs: 6 / total number of programs: 24 (25.00 %)\n"+ + "flaky programs: 4 / total number of programs: 24 (16.67 %)\n\n"+ "statistics for bar:\n"+ "\t↳ mismatches of bar / occurrences of bar: 5 / 6 (83.33 %)\n"+ "\t↳ mismatches of bar / total number of mismatches: 5 / 10 (50.00 %)\n"+ diff --git a/syz-verifier/test_utils.go b/syz-verifier/test_utils.go index 7c5df0cf5..a32c0aa29 100644 --- a/syz-verifier/test_utils.go +++ b/syz-verifier/test_utils.go @@ -24,6 +24,7 @@ func createTestServer(t *testing.T) *RPCServer { choiceTable: target.DefaultChoiceTable(), rnd: rand.New(rand.NewSource(time.Now().UnixNano())), progIdx: 3, + reruns: 1, } vrf.resultsdir = makeTestResultDirectory(t) vrf.stats = emptyTestStats() diff --git a/syz-verifier/verifier.go b/syz-verifier/verifier.go index c36d9236c..9e97046b9 100644 --- a/syz-verifier/verifier.go +++ b/syz-verifier/verifier.go @@ -22,6 +22,8 @@ type Result struct { Info ipc.ProgInfo // Crashed is set to true if a crash occurred while executing the program. Crashed bool + + RunIdx int } type ResultReport struct { @@ -67,9 +69,29 @@ func (s ReturnState) String() string { return state } +// VeifyRerun compares the results obtained from rerunning a program with what +// was reported in the initial result report. +func VerifyRerun(res []*Result, rr *ResultReport) bool { + for idx, cr := range rr.Reports { + for _, r := range res { + var state ReturnState + if r.Crashed { + state = ReturnState{Crashed: true} + } else { + ci := r.Info.Calls[idx] + state = ReturnState{Errno: ci.Errno, Flags: ci.Flags} + } + if state != cr.States[r.Pool] { + return false + } + } + } + return true +} + // Verify checks whether the Results of the same program, executed on different -// kernels are the same. If that's not the case, it returns a ResultReport -// which highlights the differences. +// kernels, are the same. If that's not the case, it returns a ResultReport, +// highlighting the differences. func Verify(res []*Result, prog *prog.Prog, s *Stats) *ResultReport { rr := &ResultReport{ Prog: string(prog.Serialize()), @@ -100,26 +122,14 @@ func Verify(res []*Result, prog *prog.Prog, s *Stats) *ResultReport { var send bool pool0 := res[0].Pool for _, cr := range rr.Reports { - cs := s.Calls[cr.Call] - - mismatch := false for _, state := range cr.States { // For each CallReport, verify whether the ReturnStates from all // the pools that executed the program are the same if state0 := cr.States[pool0]; state0 != state { cr.Mismatch = true send = true - mismatch = true - - cs.States[state] = true - cs.States[state0] = true } } - - if mismatch { - cs.Mismatches++ - s.TotalMismatches++ - } } if send { diff --git a/syz-verifier/verifier_test.go b/syz-verifier/verifier_test.go index babd87476..41344bafc 100644 --- a/syz-verifier/verifier_test.go +++ b/syz-verifier/verifier_test.go @@ -18,7 +18,7 @@ func TestVerify(t *testing.T) { name string res []*Result wantReport *ResultReport - wantStats *Stats + wantStats []*CallStats }{ { name: "only crashes", @@ -27,13 +27,6 @@ func TestVerify(t *testing.T) { makeResultCrashed(4), }, wantReport: nil, - wantStats: &Stats{ - Calls: map[string]*CallStats{ - "breaks_returns": {Name: "breaks_returns", Occurrences: 1, States: map[ReturnState]bool{}}, - "minimize$0": {Name: "minimize$0", Occurrences: 1, States: map[ReturnState]bool{}}, - "test$res0": {Name: "test$res0", Occurrences: 1, States: map[ReturnState]bool{}}, - }, - }, }, { name: "mismatches because results and crashes", @@ -62,24 +55,6 @@ func TestVerify(t *testing.T) { Mismatch: true}, }, }, - wantStats: &Stats{ - TotalMismatches: 6, - Calls: map[string]*CallStats{ - "breaks_returns": {Name: "breaks_returns", Occurrences: 1, Mismatches: 2, States: map[ReturnState]bool{ - crashedReturnState(): true, - returnState(11, 1): true, - }}, - "minimize$0": {Name: "minimize$0", Occurrences: 1, - Mismatches: 2, States: map[ReturnState]bool{ - crashedReturnState(): true, - returnState(33, 3): true, - }}, - "test$res0": {Name: "test$res0", Occurrences: 1, - Mismatches: 2, States: map[ReturnState]bool{ - crashedReturnState(): true, - returnState(22, 3): true}}, - }, - }, }, { name: "mismatches not found in results", @@ -87,13 +62,6 @@ func TestVerify(t *testing.T) { makeResult(2, []int{11, 33, 22}, []int{1, 3, 3}...), makeResult(4, []int{11, 33, 22}, []int{1, 3, 3}...)}, wantReport: nil, - wantStats: &Stats{ - Calls: map[string]*CallStats{ - "breaks_returns": {Name: "breaks_returns", Occurrences: 1, States: map[ReturnState]bool{}}, - "minimize$0": {Name: "minimize$0", Occurrences: 1, States: map[ReturnState]bool{}}, - "test$res0": {Name: "test$res0", Occurrences: 1, States: map[ReturnState]bool{}}, - }, - }, }, { name: "mismatches found in results", @@ -109,19 +77,7 @@ func TestVerify(t *testing.T) { {Call: "test$res0", States: map[int]ReturnState{1: {Errno: 2, Flags: 7}, 4: {Errno: 5, Flags: 3}}, Mismatch: true}, }, }, - wantStats: &Stats{ - TotalMismatches: 1, - Calls: map[string]*CallStats{ - "breaks_returns": {Name: "breaks_returns", Occurrences: 1, States: map[ReturnState]bool{}}, - "minimize$0": {Name: "minimize$0", Occurrences: 1, States: map[ReturnState]bool{}}, - "test$res0": {Name: "test$res0", Occurrences: 1, - Mismatches: 1, States: map[ReturnState]bool{ - {Errno: 2, Flags: 7}: true, - {Errno: 5, Flags: 3}: true}}, - }, - }, - }, - } + }} for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -135,9 +91,6 @@ func TestVerify(t *testing.T) { if diff := cmp.Diff(test.wantReport, got); diff != "" { t.Errorf("Verify report mismatch (-want +got):\n%s", diff) } - if diff := cmp.Diff(test.wantStats, stats); diff != "" { - t.Errorf("Verify stats mismatch (-want +got):\n%s", diff) - } }) } } -- cgit mrf-deployment