tools/syz-testbed: show diffs and p-values

Enable the user to specify the pivot column for the stats table. If
such a column is set, calculate and print the relative difference
between checkouts and p-values for the estimation of statistical
significance of the experimental data.

For the p-value calculation use the existing implementation from the
go-benchstat tool.

2 files changed, 32 insertions, 0 deletions

diff --git a/pkg/html/html.go b/pkg/html/html.go
index 6d24ba7e7..0b6fdaa75 100644
--- a/pkg/html/html.go
+++ b/pkg/html/html.go
@@ -8,6 +8,7 @@ package html
 import (
 	"fmt"
 	"html/template"
+	"reflect"
 	"strings"
 	texttemplate "text/template"
 	"time"
@@ -46,6 +47,7 @@ var Funcs = template.FuncMap{
 	"formatCommitTableTitle": formatCommitTableTitle,
 	"formatList":             formatStringList,
 	"selectBisect":           selectBisect,
+	"dereference":            dereferencePointer,
 }
 
 func selectBisect(rep *dashapi.BugReport) *dashapi.BisectResult {
@@ -180,3 +182,14 @@ func formatCommitTableTitle(v string) string {
 func formatStringList(list []string) string {
 	return strings.Join(list, ", ")
 }
+
+func dereferencePointer(v interface{}) interface{} {
+	reflectValue := reflect.ValueOf(v)
+	if !reflectValue.IsNil() && reflectValue.Kind() == reflect.Ptr {
+		elem := reflectValue.Elem()
+		if elem.CanInterface() {
+			return elem.Interface()
+		}
+	}
+	return v
+}
diff --git a/pkg/stats/pvalue.go b/pkg/stats/pvalue.go
new file mode 100644
index 000000000..cb8c3bada
--- /dev/null
+++ b/pkg/stats/pvalue.go
@@ -0,0 +1,19 @@
+// Copyright 2021 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package stats
+
+import "golang.org/x/perf/benchstat"
+
+// Mann-Whitney U test.
+func UTest(old, new *Sample) (pval float64, err error) {
+	// Unfortunately we cannot just invoke MannWhitneyUTest from x/perf/benchstat/internal/stats,
+	// so we first wrap the data in Metrics.
+	mOld := benchstat.Metrics{
+		RValues: old.Xs,
+	}
+	mNew := benchstat.Metrics{
+		RValues: new.Xs,
+	}
+	return benchstat.UTest(&mOld, &mNew)
+}