syz-verifier/verf: add report for mismatching results

The Verify function returns a ResultReport when errno mismatches are found.

2 files changed, 87 insertions, 71 deletions

diff --git a/syz-verifier/verf/verifier.go b/syz-verifier/verf/verifier.go
index 2678af23b..3911d3229 100644
--- a/syz-verifier/verf/verifier.go
+++ b/syz-verifier/verf/verifier.go
@@ -21,25 +21,60 @@ type Result struct {
 	Info ipc.ProgInfo
 }
 
+type ResultReport struct {
+	// Prog is the serialized program.
+	Prog string
+	// Reports contains information about each system call.
+	Reports []CallReport
+}
+
+type CallReport struct {
+	// Call is the name of the system call.
+	Call string
+	// Errno is a map between pools and the errno returned by executing the
+	// system call on a VM spawned by the respective pool.
+	Errnos map[int]int
+	// Flags is a map between pools and call flags (see pkg/ipc/ipc.go).
+	Flags map[int]ipc.CallFlags
+	// Mismatch is set to true if the returned error codes were not the same.
+	Mismatch bool
+}
+
 // Verify checks whether the Results of the same program, executed on different
-// kernels are the same. If that's the case, it returns true, otherwise it
-// returns false.
-func Verify(res []*Result, prog *prog.Prog) bool {
-	for i := 1; i < len(res); i++ {
-		if !VerifyErrnos(res[0].Info.Calls, res[i].Info.Calls) {
-			return false
+// kernels are the same. If that's not the case, it returns a ResultReport
+// which highlights the differences.
+func Verify(res []*Result, prog *prog.Prog) *ResultReport {
+	rr := &ResultReport{
+		Prog: string(prog.Serialize()),
+	}
+	c0 := res[0].Info.Calls
+	for idx, c := range c0 {
+		cr := CallReport{
+			Call:   prog.Calls[idx].Meta.Name,
+			Errnos: map[int]int{res[0].Pool: c.Errno},
+			Flags:  map[int]ipc.CallFlags{res[0].Pool: c.Flags},
 		}
+		rr.Reports = append(rr.Reports, cr)
 	}
-	return true
-}
 
-// VerifyErrnos checks whether the returned system call errnos of the same
-// program, executed on two different kernels, are the same.
-func VerifyErrnos(c1, c2 []ipc.CallInfo) bool {
-	for idx, c := range c1 {
-		if c.Errno != c2[idx].Errno {
-			return false
+	var send bool
+	for i := 1; i < len(res); i++ {
+		resi := res[i]
+		ci := resi.Info.Calls
+		for idx, c := range ci {
+			if c.Errno != c0[idx].Errno {
+				rr.Reports[idx].Mismatch = true
+				send = true
+			}
+
+			cr := rr.Reports[idx]
+			cr.Errnos[resi.Pool] = c.Errno
+			cr.Flags[resi.Pool] = c.Flags
 		}
 	}
-	return true
+
+	if send {
+		return rr
+	}
+	return nil
 }
diff --git a/syz-verifier/verf/verifier_test.go b/syz-verifier/verf/verifier_test.go
index c9a25acb5..afe2d2076 100644
--- a/syz-verifier/verf/verifier_test.go
+++ b/syz-verifier/verf/verifier_test.go
@@ -5,81 +5,62 @@ package verf
 import (
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/google/syzkaller/pkg/ipc"
+	"github.com/google/syzkaller/prog"
 )
 
+func makeResult(pool int, errnos []int, flags []int) *Result {
+	r := &Result{Pool: pool, Info: ipc.ProgInfo{Calls: []ipc.CallInfo{}}}
+	for i := range errnos {
+		r.Info.Calls = append(r.Info.Calls, ipc.CallInfo{Errno: errnos[i], Flags: ipc.CallFlags(flags[i])})
+	}
+	return r
+}
+
 func TestVerify(t *testing.T) {
+	p := "breaks_returns()\n" +
+		"minimize$0(0x1, 0x1)\n" +
+		"test$res0()\n"
 	tests := []struct {
-		name      string
-		res       []*Result
-		wantMatch bool
+		name string
+		res  []*Result
+		want *ResultReport
 	}{
 		{
-
-			name: "results should match",
+			name: "mismatches not found in results",
 			res: []*Result{
-				{2, false, ipc.ProgInfo{
-					Calls: []ipc.CallInfo{{Errno: 1}, {Errno: 2}, {Errno: 3}}, Extra: ipc.CallInfo{},
-				},
-				},
-				{4, false, ipc.ProgInfo{
-					Calls: []ipc.CallInfo{{Errno: 1}, {Errno: 2}, {Errno: 3}},
-					Extra: ipc.CallInfo{},
-				},
-				},
-			},
-
-			wantMatch: true,
+				makeResult(2, []int{11, 33, 22}, []int{1, 3, 3}),
+				makeResult(4, []int{11, 33, 22}, []int{1, 3, 3})},
+			want: nil,
 		},
 		{
-			name: "results should not match",
+			name: "mismatches found in results",
 			res: []*Result{
-				{4, false, ipc.ProgInfo{
-					Calls: []ipc.CallInfo{{Errno: 1}, {Errno: 2}, {Errno: 5}},
-				},
-				},
-				{8, false, ipc.ProgInfo{
-					Calls: []ipc.CallInfo{{Errno: 1}, {Errno: 3}, {Errno: 5}},
-				},
+				makeResult(1, []int{1, 3, 2}, []int{1, 3, 7}),
+				makeResult(4, []int{1, 3, 5}, []int{1, 3, 3}),
+			},
+			want: &ResultReport{
+				Prog: p,
+				Reports: []CallReport{
+					{Call: "breaks_returns", Errnos: map[int]int{1: 1, 4: 1}, Flags: map[int]ipc.CallFlags{1: 1, 4: 1}},
+					{Call: "minimize$0", Errnos: map[int]int{1: 3, 4: 3}, Flags: map[int]ipc.CallFlags{1: 3, 4: 3}},
+					{Call: "test$res0", Errnos: map[int]int{1: 2, 4: 5}, Flags: map[int]ipc.CallFlags{1: 7, 4: 3}, Mismatch: true},
 				},
 			},
-			wantMatch: false,
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			if got, want := Verify(test.res, nil), test.wantMatch; got != want {
-				t.Errorf("Verify: got %v, want %v", got, want)
+			target := prog.InitTargetTest(t, "test", "64")
+			prog, err := target.Deserialize([]byte(p), prog.Strict)
+			if err != nil {
+				t.Fatalf("failed to deserialise test program: %v", err)
 			}
-		})
-	}
-}
-
-func TestVerifyErrno(t *testing.T) {
-	tests := []struct {
-		name      string
-		c1, c2    []ipc.CallInfo
-		wantMatch bool
-	}{
-		{
-			name:      "errno should match",
-			c1:        []ipc.CallInfo{{Errno: 1}, {Errno: 2}, {Errno: 5}},
-			c2:        []ipc.CallInfo{{Errno: 1}, {Errno: 2}, {Errno: 5}},
-			wantMatch: true,
-		},
-		{
-			name:      "errno should not match",
-			c1:        []ipc.CallInfo{{Errno: 1}, {Errno: 2}, {Errno: 5}},
-			c2:        []ipc.CallInfo{{Errno: 1}, {Errno: 4}, {Errno: 5}},
-			wantMatch: false,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			if got, want := VerifyErrnos(test.c1, test.c2), test.wantMatch; got != want {
-				t.Errorf("VerifyErrno: got %v, want %v", got, want)
+			got := Verify(test.res, prog)
+			if diff := cmp.Diff(test.want, got); diff != "" {
+				t.Errorf("Verify mismatch (-want +got):\n%s", diff)
 			}
 		})
 	}