syz-verifier: gather stats based on ReturnState not just Errno

7 files changed, 63 insertions, 50 deletions

diff --git a/syz-verifier/main.go b/syz-verifier/main.go
index 683942871..1a34c4716 100755
--- a/syz-verifier/main.go
+++ b/syz-verifier/main.go
@@ -14,7 +14,6 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"syscall"
 	"time"
 
 	"github.com/google/syzkaller/pkg/instance"
@@ -468,11 +467,7 @@ func createReport(rr *ResultReport, pools int) []byte {
 				continue
 			}
 
-			errnoDesc := "success"
-			if state.Errno != 0 {
-				errnoDesc = syscall.Errno(state.Errno).Error()
-			}
-			data += fmt.Sprintf("\t↳ Pool: %d, Flags: %d, Errno: %d (%s)\n", i, state.Flags, state.Errno, errnoDesc)
+			data += fmt.Sprintf("\t↳ Pool: %d, %s", i, state.String())
 		}
 
 		data += "\n"
diff --git a/syz-verifier/main_test.go b/syz-verifier/main_test.go
index b021d05ae..519aa9248 100644
--- a/syz-verifier/main_test.go
+++ b/syz-verifier/main_test.go
@@ -341,9 +341,9 @@ func TestProcessResults(t *testing.T) {
 				TotalMismatches: 1,
 				Progs:           1,
 				Calls: map[string]*CallStats{
-					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[int]bool{}),
-					"test$res0":      makeCallStats("test$res0", 1, 1, map[int]bool{2: true, 5: true}),
-					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[int]bool{}),
+					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[ReturnState]bool{}),
+					"test$res0":      makeCallStats("test$res0", 1, 1, map[ReturnState]bool{{Errno: 2}: true, {Errno: 5}: true}),
+					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[ReturnState]bool{}),
 				},
 			},
 		},
@@ -356,9 +356,9 @@ func TestProcessResults(t *testing.T) {
 			wantStats: &Stats{
 				Progs: 1,
 				Calls: map[string]*CallStats{
-					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[int]bool{}),
-					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[int]bool{}),
-					"test$res0":      makeCallStats("test$res0", 1, 0, map[int]bool{}),
+					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[ReturnState]bool{}),
+					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[ReturnState]bool{}),
+					"test$res0":      makeCallStats("test$res0", 1, 0, map[ReturnState]bool{}),
 				},
 			},
 		},
@@ -462,9 +462,9 @@ func TestCleanup(t *testing.T) {
 			wantStats: &Stats{
 				Progs: 1,
 				Calls: map[string]*CallStats{
-					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[int]bool{}),
-					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[int]bool{}),
-					"test$res0":      makeCallStats("test$res0", 1, 0, map[int]bool{}),
+					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[ReturnState]bool{}),
+					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[ReturnState]bool{}),
+					"test$res0":      makeCallStats("test$res0", 1, 0, map[ReturnState]bool{}),
 				},
 			},
 			fileExists: false,
@@ -485,9 +485,9 @@ func TestCleanup(t *testing.T) {
 				TotalMismatches: 1,
 				Progs:           1,
 				Calls: map[string]*CallStats{
-					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[int]bool{}),
-					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[int]bool{}),
-					"test$res0":      makeCallStats("test$res0", 1, 1, map[int]bool{22: true, 44: true}),
+					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[ReturnState]bool{}),
+					"minimize$0":     makeCallStats("minimize$0", 1, 0, map[ReturnState]bool{}),
+					"test$res0":      makeCallStats("test$res0", 1, 1, map[ReturnState]bool{{Errno: 22}: true, {Errno: 44}: true}),
 				},
 			},
 			fileExists: true,
diff --git a/syz-verifier/stats.go b/syz-verifier/stats.go
index 5627e6342..08bae2804 100644
--- a/syz-verifier/stats.go
+++ b/syz-verifier/stats.go
@@ -9,7 +9,6 @@ import (
 	"os"
 	"os/signal"
 	"sort"
-	"syscall"
 	"time"
 
 	"github.com/google/syzkaller/prog"
@@ -36,8 +35,8 @@ type CallStats struct {
 	// Occurrences is the number of times the system call appeared in a
 	// verified program.
 	Occurrences int
-	// States stores the kernel return values that caused mismatches.
-	States map[int]bool
+	// States stores the kernel return state that caused mismatches.
+	States map[ReturnState]bool
 }
 
 // InitStats creates a stats object that will report verification
@@ -48,7 +47,9 @@ func InitStats(calls map[*prog.Syscall]bool, w io.Writer) *Stats {
 		StartTime: time.Now(),
 	}
 	for c := range calls {
-		s.Calls[c.Name] = &CallStats{Name: c.Name, States: make(map[int]bool)}
+		s.Calls[c.Name] = &CallStats{
+			Name:   c.Name,
+			States: make(map[ReturnState]bool)}
 	}
 
 	c := make(chan os.Signal)
@@ -127,19 +128,17 @@ func (s *Stats) getOrderedStats() []*CallStats {
 
 func (s *Stats) getOrderedStates(call string) []string {
 	states := s.Calls[call].States
-	ss := make([]int, 0, len(states))
+	ss := make([]ReturnState, 0, len(states))
 	for s := range states {
 		ss = append(ss, s)
 	}
-	sort.Ints(ss)
+	sort.Slice(ss, func(i, j int) bool {
+		return ss[i].Errno < ss[j].Errno
+	})
 
 	descs := make([]string, 0, len(states))
 	for _, s := range ss {
-		desc := "succes"
-		if s != 0 {
-			desc = syscall.Errno(s).Error()
-		}
-		descs = append(descs, fmt.Sprintf("%d (%s)", s, desc))
+		descs = append(descs, s.String())
 	}
 	return descs
 }
diff --git a/syz-verifier/stats_test.go b/syz-verifier/stats_test.go
index 452b10876..6cbef81f6 100644
--- a/syz-verifier/stats_test.go
+++ b/syz-verifier/stats_test.go
@@ -14,10 +14,10 @@ func dummyStats() *Stats {
 		Progs:           24,
 		TotalMismatches: 10,
 		Calls: map[string]*CallStats{
-			"foo": {"foo", 2, 8, map[int]bool{1: true, 3: true}},
-			"bar": {"bar", 5, 6, map[int]bool{10: true, 22: true}},
-			"tar": {"tar", 3, 4, map[int]bool{5: true, 17: true, 31: true}},
-			"biz": {"biz", 0, 2, map[int]bool{}},
+			"foo": {"foo", 2, 8, map[ReturnState]bool{{Errno: 1}: true, {Errno: 3}: true}},
+			"bar": {"bar", 5, 6, map[ReturnState]bool{{Errno: 10}: true, {Errno: 22}: true}},
+			"tar": {"tar", 3, 4, map[ReturnState]bool{{Errno: 5}: true, {Errno: 17}: true, {Errno: 31}: true}},
+			"biz": {"biz", 0, 2, map[ReturnState]bool{}},
 		},
 	}
 }
@@ -38,7 +38,7 @@ func TestReportCallStats(t *testing.T) {
 				"\t↳ mismatches of foo / occurrences of foo: 2 / 8 (25.00 %)\n" +
 				"\t↳ mismatches of foo / total number of mismatches: 2 / 10 (20.00 %)\n" +
 				"\t↳ 2 distinct states identified: " +
-				"[1 (operation not permitted) 3 (no such process)]\n",
+				"[Errno: 1 (operation not permitted)\n Errno: 3 (no such process)\n]\n",
 		},
 	}
 
@@ -65,17 +65,17 @@ func TestReportGlobalStats(t *testing.T) {
 			"\t↳ mismatches of bar / occurrences of bar: 5 / 6 (83.33 %)\n"+
 			"\t↳ mismatches of bar / total number of mismatches: 5 / 10 (50.00 %)\n"+
 			"\t↳ 2 distinct states identified: "+
-			"[10 (no child processes) 22 (invalid argument)]\n\n"+
+			"[Errno: 10 (no child processes)\n Errno: 22 (invalid argument)\n]\n\n"+
 			"statistics for tar:\n"+
 			"\t↳ mismatches of tar / occurrences of tar: 3 / 4 (75.00 %)\n"+
 			"\t↳ mismatches of tar / total number of mismatches: 3 / 10 (30.00 %)\n"+
 			"\t↳ 3 distinct states identified: "+
-			"[5 (input/output error) 17 (file exists) 31 (too many links)]\n\n"+
+			"[Errno: 5 (input/output error)\n Errno: 17 (file exists)\n Errno: 31 (too many links)\n]\n\n"+
 			"statistics for foo:\n"+
 			"\t↳ mismatches of foo / occurrences of foo: 2 / 8 (25.00 %)\n"+
 			"\t↳ mismatches of foo / total number of mismatches: 2 / 10 (20.00 %)\n"+
 			"\t↳ 2 distinct states identified: "+
-			"[1 (operation not permitted) 3 (no such process)]\n\n"
+			"[Errno: 1 (operation not permitted)\n Errno: 3 (no such process)\n]\n\n"
 
 	if diff := cmp.Diff(want, got); diff != "" {
 		t.Errorf("s.ReportGlobalStats mismatch (-want +got):\n%s", diff)
diff --git a/syz-verifier/test_utils.go b/syz-verifier/test_utils.go
index 0ff35d874..bc604e4ed 100644
--- a/syz-verifier/test_utils.go
+++ b/syz-verifier/test_utils.go
@@ -74,14 +74,14 @@ func makeResult(pool int, errnos []int, flags ...int) *Result {
 func emptyTestStats() *Stats {
 	return &Stats{
 		Calls: map[string]*CallStats{
-			"breaks_returns": {Name: "breaks_returns", States: map[int]bool{}},
-			"minimize$0":     {Name: "minimize$0", States: map[int]bool{}},
-			"test$res0":      {Name: "test$res0", States: map[int]bool{}},
+			"breaks_returns": {Name: "breaks_returns", States: map[ReturnState]bool{}},
+			"minimize$0":     {Name: "minimize$0", States: map[ReturnState]bool{}},
+			"test$res0":      {Name: "test$res0", States: map[ReturnState]bool{}},
 		},
 	}
 }
 
-func makeCallStats(name string, occurrences, mismatches int, states map[int]bool) *CallStats {
+func makeCallStats(name string, occurrences, mismatches int, states map[ReturnState]bool) *CallStats {
 	return &CallStats{Name: name,
 		Occurrences: occurrences,
 		Mismatches:  mismatches,
diff --git a/syz-verifier/verifier.go b/syz-verifier/verifier.go
index 884609b6a..f86106868 100644
--- a/syz-verifier/verifier.go
+++ b/syz-verifier/verifier.go
@@ -4,6 +4,9 @@
 package main
 
 import (
+	"fmt"
+	"syscall"
+
 	"github.com/google/syzkaller/pkg/ipc"
 	"github.com/google/syzkaller/prog"
 )
@@ -43,6 +46,19 @@ type ReturnState struct {
 	Flags ipc.CallFlags
 }
 
+func (s ReturnState) String() string {
+	state := ""
+	if s.Flags != 0 {
+		state += fmt.Sprintf("Flags: %d, ", s.Flags)
+	}
+	errDesc := "success"
+	if s.Errno != 0 {
+		errDesc = syscall.Errno(s.Errno).Error()
+	}
+	state += fmt.Sprintf("Errno: %d (%s)\n", s.Errno, errDesc)
+	return state
+}
+
 // Verify checks whether the Results of the same program, executed on different
 // kernels are the same. If that's not the case, it returns a ResultReport
 // which highlights the differences.
@@ -75,14 +91,14 @@ func Verify(res []*Result, prog *prog.Prog, s *Stats) *ResultReport {
 		for _, state := range cr.States {
 			// For each CallReport verify the ReturnStates from all the pools
 			// that executed the program are the same
-			if errno0 := cr.States[pool0].Errno; errno0 != state.Errno {
+			if state0 := cr.States[pool0]; state0 != state {
 				cr.Mismatch = true
 				send = true
 
 				s.TotalMismatches++
 				cs.Mismatches++
-				cs.States[state.Errno] = true
-				cs.States[errno0] = true
+				cs.States[state] = true
+				cs.States[state0] = true
 			}
 		}
 	}
diff --git a/syz-verifier/verifier_test.go b/syz-verifier/verifier_test.go
index 8e8b4f3b2..254e28a25 100644
--- a/syz-verifier/verifier_test.go
+++ b/syz-verifier/verifier_test.go
@@ -28,9 +28,9 @@ func TestVerify(t *testing.T) {
 			wantReport: nil,
 			wantStats: &Stats{
 				Calls: map[string]*CallStats{
-					"breaks_returns": {Name: "breaks_returns", Occurrences: 1, States: map[int]bool{}},
-					"minimize$0":     {Name: "minimize$0", Occurrences: 1, States: map[int]bool{}},
-					"test$res0":      {Name: "test$res0", Occurrences: 1, States: map[int]bool{}},
+					"breaks_returns": {Name: "breaks_returns", Occurrences: 1, States: map[ReturnState]bool{}},
+					"minimize$0":     {Name: "minimize$0", Occurrences: 1, States: map[ReturnState]bool{}},
+					"test$res0":      {Name: "test$res0", Occurrences: 1, States: map[ReturnState]bool{}},
 				},
 			},
 		},
@@ -51,9 +51,12 @@ func TestVerify(t *testing.T) {
 			wantStats: &Stats{
 				TotalMismatches: 1,
 				Calls: map[string]*CallStats{
-					"breaks_returns": {Name: "breaks_returns", Occurrences: 1, States: map[int]bool{}},
-					"minimize$0":     {Name: "minimize$0", Occurrences: 1, States: map[int]bool{}},
-					"test$res0":      {Name: "test$res0", Occurrences: 1, Mismatches: 1, States: map[int]bool{2: true, 5: true}},
+					"breaks_returns": {Name: "breaks_returns", Occurrences: 1, States: map[ReturnState]bool{}},
+					"minimize$0":     {Name: "minimize$0", Occurrences: 1, States: map[ReturnState]bool{}},
+					"test$res0": {Name: "test$res0", Occurrences: 1,
+						Mismatches: 1, States: map[ReturnState]bool{
+							{Errno: 2, Flags: 7}: true,
+							{Errno: 5, Flags: 3}: true}},
 				},
 			},
 		},