syz-verifier: move stats gathering in the Verify function

5 files changed, 83 insertions, 75 deletions

diff --git a/syz-verifier/main.go b/syz-verifier/main.go
index 741709f51..09049a23f 100755
--- a/syz-verifier/main.go
+++ b/syz-verifier/main.go
@@ -390,10 +390,7 @@ func (srv *RPCServer) newResult(res *verf.Result, prog *progInfo) bool {
 // in th workdir/results directory. If writing the results fails, it returns an
 // error.
 func (vrf *Verifier) processResults(res []*verf.Result, prog *prog.Prog) {
-	for _, c := range prog.Calls {
-		vrf.stats.Calls[c.Meta.Name].Occurrences++
-	}
-	rr := verf.Verify(res, prog)
+	rr := verf.Verify(res, prog, vrf.stats)
 	if rr == nil {
 		return
 	}
@@ -418,7 +415,7 @@ func (vrf *Verifier) processResults(res []*verf.Result, prog *prog.Prog) {
 	}
 
 	err := osutil.WriteFile(filepath.Join(vrf.resultsdir,
-		fmt.Sprintf("result-%d", oldest)), createReport(rr, len(vrf.pools), vrf.stats))
+		fmt.Sprintf("result-%d", oldest)), createReport(rr, len(vrf.pools)))
 	if err != nil {
 		log.Printf("failed to write result-%d file, err %v", oldest, err)
 	}
@@ -426,18 +423,15 @@ func (vrf *Verifier) processResults(res []*verf.Result, prog *prog.Prog) {
 	log.Printf("result-%d written successfully", oldest)
 }
 
-func createReport(rr *verf.ResultReport, pools int, s *stats.Stats) []byte {
+func createReport(rr *verf.ResultReport, pools int) []byte {
 	calls := strings.Split(rr.Prog, "\n")
 	calls = calls[:len(calls)-1]
 
 	data := "ERRNO mismatches found for program:\n\n"
 	for idx, cr := range rr.Reports {
-		cs := s.Calls[cr.Call]
 		tick := "[=]"
 		if cr.Mismatch {
 			tick = "[!]"
-			cs.Mismatches++
-			s.TotalMismatches++
 		}
 		data += fmt.Sprintf("%s %s\n", tick, calls[idx])
 
diff --git a/syz-verifier/main_test.go b/syz-verifier/main_test.go
index 1964ded37..9ba08c955 100644
--- a/syz-verifier/main_test.go
+++ b/syz-verifier/main_test.go
@@ -58,13 +58,20 @@ func getTestProgram(t *testing.T) *prog.Prog {
 func getTestStats() *stats.Stats {
 	return &stats.Stats{
 		Calls: map[string]*stats.CallStats{
-			"breaks_returns": {Name: "breaks_returns"},
-			"minimize$0":     {Name: "minimize$0"},
-			"test$res0":      {Name: "test$res0"},
+			"breaks_returns": {Name: "breaks_returns", States: map[int]bool{}},
+			"minimize$0":     {Name: "minimize$0", States: map[int]bool{}},
+			"test$res0":      {Name: "test$res0", States: map[int]bool{}},
 		},
 	}
 }
 
+func makeCallStats(name string, occurrences, mismatches int, states map[int]bool) *stats.CallStats {
+	return &stats.CallStats{Name: name,
+		Occurrences: occurrences,
+		Mismatches:  mismatches,
+		States:      states}
+}
+
 func makeTestResultDirectory(t *testing.T) string {
 	resultsdir := "test"
 	err := osutil.MkdirAll(resultsdir)
@@ -411,17 +418,9 @@ func TestProcessResults(t *testing.T) {
 			wantStats: &stats.Stats{
 				TotalMismatches: 1,
 				Calls: map[string]*stats.CallStats{
-					"breaks_returns": {
-						Name:        "breaks_returns",
-						Occurrences: 1},
-					"minimize$0": {
-						Name:        "minimize$0",
-						Occurrences: 1},
-					"test$res0": {
-						Name:        "test$res0",
-						Occurrences: 1,
-						Mismatches:  1,
-					},
+					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[int]bool{1: true}),
+					"test$res0":      makeCallStats("test$res0", 1, 1, map[int]bool{2: true, 5: true}),
+					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[int]bool{3: true}),
 				},
 			},
 		},
@@ -433,16 +432,9 @@ func TestProcessResults(t *testing.T) {
 			},
 			wantStats: &stats.Stats{
 				Calls: map[string]*stats.CallStats{
-					"breaks_returns": {
-						Name:        "breaks_returns",
-						Occurrences: 1},
-					"minimize$0": {
-						Name:        "minimize$0",
-						Occurrences: 1},
-					"test$res0": {
-						Name:        "test$res0",
-						Occurrences: 1,
-					},
+					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[int]bool{11: true}),
+					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[int]bool{33: true}),
+					"test$res0":      makeCallStats("test$res0", 1, 0, map[int]bool{22: true}),
 				},
 			},
 		},
@@ -484,15 +476,7 @@ func TestCreateReport(t *testing.T) {
 				Flags: map[int]ipc.CallFlags{1: 7, 2: 3, 3: 1}, Mismatch: true},
 		},
 	}
-	got := string(createReport(&rr, 3,
-		&stats.Stats{
-			TotalMismatches: 10,
-			Calls: map[string]*stats.CallStats{
-				"test$res0":      {Name: "test$res0", Mismatches: 2, Occurrences: 4},
-				"breaks_returns": {Name: "breaks_returns", Mismatches: 0, Occurrences: 3},
-				"minimize$0":     {Name: "minimize$0", Mismatches: 1, Occurrences: 2},
-			},
-		}))
+	got := string(createReport(&rr, 3))
 	want := "ERRNO mismatches found for program:\n\n" +
 		"[=] breaks_returns()\n" +
 		"\t↳ Pool: 1, Errno: 1, Flag: 1\n" +
@@ -529,13 +513,7 @@ func TestCleanup(t *testing.T) {
 				idx:  4,
 				left: map[int]bool{1: true, 2: true},
 			},
-			wantStats: &stats.Stats{
-				Calls: map[string]*stats.CallStats{
-					"breaks_returns": {Name: "breaks_returns"},
-					"minimize$0":     {Name: "minimize$0"},
-					"test$res0":      {Name: "test$res0"},
-				},
-			},
+			wantStats:  getTestStats(),
 			fileExists: false,
 		},
 		{
@@ -552,9 +530,9 @@ func TestCleanup(t *testing.T) {
 				}},
 			wantStats: &stats.Stats{
 				Calls: map[string]*stats.CallStats{
-					"breaks_returns": {Name: "breaks_returns", Occurrences: 1},
-					"minimize$0":     {Name: "minimize$0", Occurrences: 1},
-					"test$res0":      {Name: "test$res0", Occurrences: 1},
+					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[int]bool{11: true}),
+					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[int]bool{33: true}),
+					"test$res0":      makeCallStats("test$res0", 1, 0, map[int]bool{22: true}),
 				},
 			},
 			fileExists: false,
@@ -574,9 +552,9 @@ func TestCleanup(t *testing.T) {
 			wantStats: &stats.Stats{
 				TotalMismatches: 1,
 				Calls: map[string]*stats.CallStats{
-					"breaks_returns": {Name: "breaks_returns", Occurrences: 1},
-					"minimize$0":     {Name: "minimize$0", Occurrences: 1},
-					"test$res0":      {Name: "test$res0", Occurrences: 1, Mismatches: 1},
+					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[int]bool{11: true}),
+					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[int]bool{33: true}),
+					"test$res0":      makeCallStats("test$res0", 1, 1, map[int]bool{22: true, 44: true}),
 				},
 			},
 			fileExists: true,
@@ -591,13 +569,7 @@ func TestCleanup(t *testing.T) {
 						makeResult(2, []int{11, 33, 22}),
 					},
 				}},
-			wantStats: &stats.Stats{
-				Calls: map[string]*stats.CallStats{
-					"breaks_returns": {Name: "breaks_returns"},
-					"minimize$0":     {Name: "minimize$0"},
-					"test$res0":      {Name: "test$res0"},
-				},
-			},
+			wantStats:  getTestStats(),
 			wantProg:   nil,
 			fileExists: false,
 		},
diff --git a/syz-verifier/stats/stats.go b/syz-verifier/stats/stats.go
index f0607f63a..aba66fc56 100644
--- a/syz-verifier/stats/stats.go
+++ b/syz-verifier/stats/stats.go
@@ -34,7 +34,7 @@ type CallStats struct {
 	// Occurrences is the number of times the system call appeared in a
 	// verified program.
 	Occurrences int
-	// States stores all possible kernel return values identified for the 
+	// States stores all possible kernel return values identified for the
 	// system call.
 	States map[int]bool
 }
@@ -44,7 +44,7 @@ type CallStats struct {
 func InitStats(calls map[*prog.Syscall]bool, w io.Writer) *Stats {
 	s := &Stats{Calls: make(map[string]*CallStats)}
 	for c := range calls {
-		s.Calls[c.Name] = &CallStats{Name: c.Name}
+		s.Calls[c.Name] = &CallStats{Name: c.Name, States: make(map[int]bool)}
 	}
 
 	c := make(chan os.Signal)
diff --git a/syz-verifier/verf/verifier.go b/syz-verifier/verf/verifier.go
index d037f566f..1dea156ad 100644
--- a/syz-verifier/verf/verifier.go
+++ b/syz-verifier/verf/verifier.go
@@ -8,6 +8,7 @@ package verf
 import (
 	"github.com/google/syzkaller/pkg/ipc"
 	"github.com/google/syzkaller/prog"
+	"github.com/google/syzkaller/syz-verifier/stats"
 )
 
 // Result stores the results of executing a program.
@@ -42,18 +43,23 @@ type CallReport struct {
 // Verify checks whether the Results of the same program, executed on different
 // kernels are the same. If that's not the case, it returns a ResultReport
 // which highlights the differences.
-func Verify(res []*Result, prog *prog.Prog) *ResultReport {
+func Verify(res []*Result, prog *prog.Prog, s *stats.Stats) *ResultReport {
 	rr := &ResultReport{
 		Prog: string(prog.Serialize()),
 	}
 	c0 := res[0].Info.Calls
 	for idx, c := range c0 {
+		call := prog.Calls[idx].Meta.Name
 		cr := CallReport{
-			Call:   prog.Calls[idx].Meta.Name,
+			Call:   call,
 			Errnos: map[int]int{res[0].Pool: c.Errno},
 			Flags:  map[int]ipc.CallFlags{res[0].Pool: c.Flags},
 		}
+
 		rr.Reports = append(rr.Reports, cr)
+		cs := s.Calls[call]
+		cs.Occurrences++
+		cs.States[c.Errno] = true
 	}
 
 	var send bool
@@ -61,12 +67,17 @@ func Verify(res []*Result, prog *prog.Prog) *ResultReport {
 		resi := res[i]
 		ci := resi.Info.Calls
 		for idx, c := range ci {
+			cr := rr.Reports[idx]
+			cs := s.Calls[cr.Call]
 			if c.Errno != c0[idx].Errno {
 				rr.Reports[idx].Mismatch = true
 				send = true
+
+				s.TotalMismatches++
+				cs.Mismatches++
+				cs.States[c.Errno] = true
 			}
 
-			cr := rr.Reports[idx]
 			cr.Errnos[resi.Pool] = c.Errno
 			cr.Flags[resi.Pool] = c.Flags
 		}
diff --git a/syz-verifier/verf/verifier_test.go b/syz-verifier/verf/verifier_test.go
index 54dcfe2d4..6494c32b4 100644
--- a/syz-verifier/verf/verifier_test.go
+++ b/syz-verifier/verf/verifier_test.go
@@ -8,6 +8,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/syzkaller/pkg/ipc"
 	"github.com/google/syzkaller/prog"
+	"github.com/google/syzkaller/syz-verifier/stats"
 )
 
 func makeResult(pool int, errnos []int, flags []int) *Result {
@@ -18,21 +19,39 @@ func makeResult(pool int, errnos []int, flags []int) *Result {
 	return r
 }
 
+func getTestStats() *stats.Stats {
+	return &stats.Stats{
+		Calls: map[string]*stats.CallStats{
+			"breaks_returns": {Name: "breaks_returns", States: map[int]bool{}},
+			"minimize$0":     {Name: "minimize$0", States: map[int]bool{}},
+			"test$res0":      {Name: "test$res0", States: map[int]bool{}},
+		},
+	}
+}
+
 func TestVerify(t *testing.T) {
 	p := "breaks_returns()\n" +
 		"minimize$0(0x1, 0x1)\n" +
 		"test$res0()\n"
 	tests := []struct {
-		name string
-		res  []*Result
-		want *ResultReport
+		name       string
+		res        []*Result
+		wantReport *ResultReport
+		wantStats  *stats.Stats
 	}{
 		{
 			name: "mismatches not found in results",
 			res: []*Result{
 				makeResult(2, []int{11, 33, 22}, []int{1, 3, 3}),
 				makeResult(4, []int{11, 33, 22}, []int{1, 3, 3})},
-			want: nil,
+			wantReport: nil,
+			wantStats: &stats.Stats{
+				Calls: map[string]*stats.CallStats{
+					"breaks_returns": {Name: "breaks_returns", Occurrences: 1, States: map[int]bool{11: true}},
+					"minimize$0":     {Name: "minimize$0", Occurrences: 1, States: map[int]bool{33: true}},
+					"test$res0":      {Name: "test$res0", Occurrences: 1, States: map[int]bool{22: true}},
+				},
+			},
 		},
 		{
 			name: "mismatches found in results",
@@ -40,7 +59,7 @@ func TestVerify(t *testing.T) {
 				makeResult(1, []int{1, 3, 2}, []int{1, 3, 7}),
 				makeResult(4, []int{1, 3, 5}, []int{1, 3, 3}),
 			},
-			want: &ResultReport{
+			wantReport: &ResultReport{
 				Prog: p,
 				Reports: []CallReport{
 					{Call: "breaks_returns", Errnos: map[int]int{1: 1, 4: 1},
@@ -51,6 +70,14 @@ func TestVerify(t *testing.T) {
 						Flags: map[int]ipc.CallFlags{1: 7, 4: 3}, Mismatch: true},
 				},
 			},
+			wantStats: &stats.Stats{
+				TotalMismatches: 1,
+				Calls: map[string]*stats.CallStats{
+					"breaks_returns": {Name: "breaks_returns", Occurrences: 1, States: map[int]bool{1: true}},
+					"minimize$0":     {Name: "minimize$0", Occurrences: 1, States: map[int]bool{3: true}},
+					"test$res0":      {Name: "test$res0", Occurrences: 1, Mismatches: 1, States: map[int]bool{2: true, 5: true}},
+				},
+			},
 		},
 	}
 
@@ -61,9 +88,13 @@ func TestVerify(t *testing.T) {
 			if err != nil {
 				t.Fatalf("failed to deserialise test program: %v", err)
 			}
-			got := Verify(test.res, prog)
-			if diff := cmp.Diff(test.want, got); diff != "" {
-				t.Errorf("Verify mismatch (-want +got):\n%s", diff)
+			stats := getTestStats()
+			got := Verify(test.res, prog, stats)
+			if diff := cmp.Diff(test.wantReport, got); diff != "" {
+				t.Errorf("Verify report mismatch (-want +got):\n%s", diff)
+			}
+			if diff := cmp.Diff(test.wantStats, stats); diff != "" {
+				t.Errorf("Verify stats mismatch (-want +got):\n%s", diff)
 			}
 		})
 	}