syz-verifier: improve statistics (#3038)

s/TotalMismatches/TotalCallMismatches/ for readability.
Add ExecErrorProgs to count failures.

5 files changed, 32 insertions, 28 deletions

diff --git a/syz-verifier/monitoring_api.go b/syz-verifier/monitoring_api.go
index af852d02a..1c95f9d26 100644
--- a/syz-verifier/monitoring_api.go
+++ b/syz-verifier/monitoring_api.go
@@ -43,24 +43,26 @@ func (monitor *Monitor) initHTTPHandlers() {
 
 // statsJSON provides information for the "/api/stats.json" render.
 type statsJSON struct {
-	StartTime        time.Time
-	TotalMismatches  int64
-	TotalProgs       int64
-	FlakyProgs       int64
-	MismatchingProgs int64
-	AverExecSpeed    int64
+	StartTime           time.Time
+	TotalCallMismatches int64
+	TotalProgs          int64
+	ExecErrorProgs      int64
+	FlakyProgs          int64
+	MismatchingProgs    int64
+	AverExecSpeed       int64
 }
 
 // handleStats renders the statsJSON object.
 func (monitor *Monitor) renderStats() interface{} {
 	stats := monitor.externalStats
 	return &statsJSON{
-		StartTime:        stats.StartTime,
-		TotalMismatches:  stats.TotalMismatches,
-		TotalProgs:       stats.TotalProgs,
-		FlakyProgs:       stats.FlakyProgs,
-		MismatchingProgs: stats.MismatchingProgs,
-		AverExecSpeed:    60 * stats.TotalProgs / int64(1+time.Since(stats.StartTime).Seconds()),
+		StartTime:           stats.StartTime,
+		TotalCallMismatches: stats.TotalCallMismatches,
+		TotalProgs:          stats.TotalProgs,
+		ExecErrorProgs:      stats.ExecErrorProgs,
+		FlakyProgs:          stats.FlakyProgs,
+		MismatchingProgs:    stats.MismatchingProgs,
+		AverExecSpeed:       60 * stats.TotalProgs / int64(1+time.Since(stats.StartTime).Seconds()),
 	}
 }
 
diff --git a/syz-verifier/stats.go b/syz-verifier/stats.go
index 17a443315..91fd7538b 100644
--- a/syz-verifier/stats.go
+++ b/syz-verifier/stats.go
@@ -16,12 +16,13 @@ import (
 // of the verification process.
 type Stats struct {
 	// Calls stores statistics for all supported system calls.
-	Calls            map[string]*CallStats
-	TotalMismatches  int64
-	TotalProgs       int64
-	FlakyProgs       int64
-	MismatchingProgs int64
-	StartTime        time.Time
+	Calls               map[string]*CallStats
+	TotalCallMismatches int64
+	TotalProgs          int64
+	ExecErrorProgs      int64
+	FlakyProgs          int64
+	MismatchingProgs    int64
+	StartTime           time.Time
 }
 
 // CallStats stores information used to generate statistics for the
@@ -65,7 +66,7 @@ func (stats *Stats) GetTextDescription(deltaTime float64) string {
 
 	tc := stats.totalCallsExecuted()
 	fmt.Fprintf(&result, "total number of mismatches / total number of calls "+
-		"executed: %d / %d (%0.2f %%)\n\n", stats.TotalMismatches, tc, getPercentage(stats.TotalMismatches, tc))
+		"executed: %d / %d (%0.2f %%)\n\n", stats.TotalCallMismatches, tc, getPercentage(stats.TotalCallMismatches, tc))
 	fmt.Fprintf(&result, "programs / minute: %0.2f\n\n", float64(stats.TotalProgs)/deltaTime)
 	fmt.Fprintf(&result, "true mismatching programs: %d / total number of programs: %d (%0.2f %%)\n",
 		stats.MismatchingProgs, stats.TotalProgs, getPercentage(stats.MismatchingProgs, stats.TotalProgs))
@@ -91,8 +92,8 @@ func (stats *Stats) getCallStatsTextDescription(call string) string {
 		"\t↳ mismatches of %s / total number of mismatches: "+
 		"%d / %d (%0.2f %%)\n"+
 		"\t↳ %d distinct states identified: %v\n", syscallName, syscallName, syscallName, mismatches, occurrences,
-		getPercentage(mismatches, occurrences), syscallName, mismatches, stats.TotalMismatches,
-		getPercentage(mismatches, stats.TotalMismatches), len(syscallStat.States), stats.getOrderedStates(syscallName))
+		getPercentage(mismatches, occurrences), syscallName, mismatches, stats.TotalCallMismatches,
+		getPercentage(mismatches, stats.TotalCallMismatches), len(syscallStat.States), stats.getOrderedStates(syscallName))
 }
 
 func (stats *Stats) totalCallsExecuted() int64 {
diff --git a/syz-verifier/stats_test.go b/syz-verifier/stats_test.go
index e553e3827..a7f8cdec8 100644
--- a/syz-verifier/stats_test.go
+++ b/syz-verifier/stats_test.go
@@ -10,10 +10,10 @@ import (
 
 func dummyStats() *Stats {
 	return &Stats{
-		TotalProgs:       24,
-		TotalMismatches:  10,
-		FlakyProgs:       4,
-		MismatchingProgs: 6,
+		TotalProgs:          24,
+		TotalCallMismatches: 10,
+		FlakyProgs:          4,
+		MismatchingProgs:    6,
 		Calls: map[string]*CallStats{
 			"foo": {"foo", 2, 8, map[ReturnState]bool{
 				returnState(1, 7): true,
diff --git a/syz-verifier/verifier.go b/syz-verifier/verifier.go
index 1ad9eaab9..4bcb0fa8a 100644
--- a/syz-verifier/verifier.go
+++ b/syz-verifier/verifier.go
@@ -143,6 +143,7 @@ func (vrf *Verifier) TestProgram(prog *prog.Prog) (result []*ExecResult) {
 	for i, env := range steps {
 		stepRes, err := vrf.Run(prog, env)
 		if err != nil {
+			atomic.AddInt64(&vrf.stats.ExecErrorProgs, 1)
 			return
 		}
 		vrf.AddCallsExecutionStat(stepRes, prog)
@@ -217,7 +218,7 @@ func (vrf *Verifier) SetPrintStatAtSIGINT() error {
 		defer os.Exit(0)
 
 		totalExecutionTime := time.Since(vrf.stats.StartTime).Minutes()
-		if vrf.stats.TotalMismatches < 0 {
+		if vrf.stats.TotalCallMismatches < 0 {
 			fmt.Fprint(vrf.statsWrite, "No mismatches occurred until syz-verifier was stopped.")
 		} else {
 			fmt.Fprintf(vrf.statsWrite, "%s", vrf.stats.GetTextDescription(totalExecutionTime))
@@ -314,7 +315,7 @@ func (vrf *Verifier) AddCallsExecutionStat(results []*ExecResult, program *prog.
 			continue
 		}
 		atomic.AddInt64(&vrf.stats.Calls[cr.Call].Mismatches, 1)
-		atomic.AddInt64(&vrf.stats.TotalMismatches, 1)
+		atomic.AddInt64(&vrf.stats.TotalCallMismatches, 1)
 		for _, state := range cr.States {
 			if state0 := cr.States[0]; state0 != state {
 				vrf.stats.Calls[cr.Call].States[state] = true
diff --git a/syz-verifier/verifier_test.go b/syz-verifier/verifier_test.go
index 0c325d4d7..fd2167b1e 100644
--- a/syz-verifier/verifier_test.go
+++ b/syz-verifier/verifier_test.go
@@ -205,7 +205,7 @@ func TestSaveDiffResults(t *testing.T) {
 			},
 			wantExist: true,
 			wantStats: &Stats{
-				TotalMismatches: 1,
+				TotalCallMismatches: 1,
 				Calls: map[string]*CallStats{
 					"breaks_returns": makeCallStats("breaks_returns", 1, 0, map[ReturnState]bool{}),
 					"test$res0":      makeCallStats("test$res0", 1, 1, map[ReturnState]bool{{Errno: 2}: true, {Errno: 5}: true}),