syz-verifier/stats: create utility to gather statistics

2 files changed, 185 insertions, 0 deletions

diff --git a/syz-verifier/stats/stats.go b/syz-verifier/stats/stats.go
new file mode 100644
index 000000000..0a96b7f71
--- /dev/null
+++ b/syz-verifier/stats/stats.go
@@ -0,0 +1,111 @@
+// Copyright 2021 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+// Package stats contains utilities that aid gathering statistics of
+// system call mismatches in the verified programs.
+package stats
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"sort"
+
+	"github.com/google/syzkaller/prog"
+)
+
+// Stats encapsulates data for creating statistics related to the results
+// of the verification process.
+type Stats struct {
+	// Calls stores statistics for all supported system calls.
+	Calls           map[string]*CallStats
+	TotalMismatches int
+}
+
+// CallStats stores information used to generate statistics for the
+// system call.
+type CallStats struct {
+	// Name is the system call name.
+	Name string
+	// Mismatches stores the number of errno mismatches identifed in the
+	// verified programs for this system call.
+	Mismatches int
+	// Occurrences is the number of times the system call appeared in a
+	// verified program.
+	Occurrences int
+}
+
+// InitStats creates a stats object that will report verification
+// statistics when an os.Interrupt occurs.
+func InitStats(calls map[*prog.Syscall]bool, w io.Writer) *Stats {
+	s := &Stats{Calls: make(map[string]*CallStats)}
+	for c := range calls {
+		s.Calls[c.Name] = &CallStats{Name: c.Name}
+	}
+
+	c := make(chan os.Signal)
+	signal.Notify(c, os.Interrupt)
+	go func() {
+		<-c
+		s.ReportGlobalStats(w)
+		os.Exit(0)
+	}()
+
+	return s
+}
+
+// ReportCallStats creates a report with the current statistics for call.
+func (s *Stats) ReportCallStats(call string) string {
+	cs, ok := s.Calls[call]
+	if !ok {
+		return ""
+	}
+	name, m, o := cs.Name, cs.Mismatches, cs.Occurrences
+	data := fmt.Sprintf("statistics for %s:\n"+
+		"\t↳ mismatches of %s / occurrences of %s: %d / %d (%0.2f %%)\n"+
+		"\t↳ mismatches of %s / total number of mismatches: "+
+		"%d / %d (%0.2f %%)\n", name, name, name, m, o,
+		getPercentage(m, o), name, m, s.TotalMismatches, getPercentage(m, s.TotalMismatches))
+	return data
+}
+
+func getPercentage(value, total int) float64 {
+	return float64(value) / float64(total) * 100
+}
+
+// ReportGlobalStats creates a report with statistics about all the
+// supported system calls for which errno mismatches were identified in
+// the verified programs, shown in decreasing order.
+func (s *Stats) ReportGlobalStats(w io.Writer) {
+	tc := s.totalCallsExecuted()
+	fmt.Fprintf(w, "total number of mismatches / total number of calls "+
+		"executed: %d / %d (%0.2f %%)\n\n", s.TotalMismatches, tc, getPercentage(s.TotalMismatches, tc))
+	cs := s.getOrderedStats()
+	for _, c := range cs {
+		fmt.Fprintf(w, "%s\n", s.ReportCallStats(c.Name))
+	}
+}
+
+func (s *Stats) totalCallsExecuted() int {
+	t := 0
+	for _, cs := range s.Calls {
+		t += cs.Occurrences
+	}
+	return t
+}
+
+func (s *Stats) getOrderedStats() []*CallStats {
+	css := make([]*CallStats, 0)
+	for _, cs := range s.Calls {
+		if cs.Mismatches > 0 {
+			css = append(css, cs)
+		}
+	}
+
+	sort.Slice(css, func(i, j int) bool {
+		return css[i].Mismatches > css[j].Mismatches
+	})
+
+	return css
+}
diff --git a/syz-verifier/stats/stats_test.go b/syz-verifier/stats/stats_test.go
new file mode 100644
index 000000000..4bc3f9149
--- /dev/null
+++ b/syz-verifier/stats/stats_test.go
@@ -0,0 +1,74 @@
+// Copyright 2021 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package stats
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func getTestStats() *Stats {
+	return &Stats{
+		TotalMismatches: 11,
+		Calls: map[string]*CallStats{
+			"foo": {"foo", 2, 8},
+			"bar": {"bar", 5, 6},
+			"tar": {"tar", 3, 4},
+			"biz": {"biz", 0, 2},
+		},
+	}
+}
+
+func TestReportCallStats(t *testing.T) {
+	tests := []struct {
+		name, call, report string
+	}{
+		{
+			name:   "report for unsupported call",
+			call:   "read",
+			report: "",
+		},
+		{
+			name: "report for supported call",
+			call: "foo",
+			report: "statistics for foo:\n" +
+				"\t↳ mismatches of foo / occurrences of foo: 2 / 8 (25.00 %)\n" +
+				"\t↳ mismatches of foo / total number of mismatches: 2 / 11 (18.18 %)\n",
+		},
+	}
+
+	for _, test := range tests {
+		s := getTestStats()
+		t.Run(test.name, func(t *testing.T) {
+			got, want := s.ReportCallStats(test.call), test.report
+			if diff := cmp.Diff(got, want); diff != "" {
+				t.Errorf("s.ReportCallStats mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+func TestReportGlobalStats(t *testing.T) {
+	s := getTestStats()
+	out := bytes.Buffer{}
+	s.ReportGlobalStats(&out)
+	got, want := out.String(),
+		"total number of mismatches / total number of calls "+
+			"executed: 11 / 20 (55.00 %)\n\n"+
+			"statistics for bar:\n"+
+			"\t↳ mismatches of bar / occurrences of bar: 5 / 6 (83.33 %)\n"+
+			"\t↳ mismatches of bar / total number of mismatches: 5 / 11 (45.45 %)\n\n"+
+			"statistics for tar:\n"+
+			"\t↳ mismatches of tar / occurrences of tar: 3 / 4 (75.00 %)\n"+
+			"\t↳ mismatches of tar / total number of mismatches: 3 / 11 (27.27 %)\n\n"+
+			"statistics for foo:\n"+
+			"\t↳ mismatches of foo / occurrences of foo: 2 / 8 (25.00 %)\n"+
+			"\t↳ mismatches of foo / total number of mismatches: 2 / 11 (18.18 %)\n\n"
+
+	if diff := cmp.Diff(got, want); diff != "" {
+		t.Errorf("s.ReportGlobalStats mismatch (-want +got):\n%s", diff)
+	}
+}