tools/syz-testbed: show diffs and p-values

Enable the user to specify the pivot column for the stats table. If such a column is set, calculate and print the relative difference between checkouts and p-values for the estimation of statistical significance of the experimental data. For the p-value calculation use the existing implementation from the go-benchstat tool.
author: Aleksandr Nogikh <nogikh@google.com> 2021-11-26 16:15:37 +0000
committer: Aleksandr Nogikh <wp32pw@gmail.com> 2021-12-06 14:29:36 +0100
commit: 9bc82119f890ecaa556935301dc721f90100d2dd (patch)
tree: 6812ecfd10fabb319f3c502cd69b9b7f443d9025 /tools/syz-testbed
parent: 8a9bfbcda821c7c38dba195d7ba4a5f34463b49b (diff)
2 files changed, 131 insertions, 24 deletions
diff --git a/tools/syz-testbed/html.go b/tools/syz-testbed/html.go
index 39d39bcb3..396265dc5 100644
--- a/tools/syz-testbed/html.go
+++ b/tools/syz-testbed/html.go
@@ -11,6 +11,7 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"time"
 
@@ -120,14 +121,21 @@ func (ctx *TestbedContext) httpGraph(w http.ResponseWriter, r *http.Request) {
 	w.Write(data)
 }
 
+type uiTable struct {
+	Table     *Table
+	ColumnURL func(string) string
+	RowURL    func(string) string
+	Extra     bool
+}
+
 type uiStatView struct {
 	Name  string
-	Table *Table
+	Table uiTable
 }
 
 type uiMainPage struct {
 	Name       string
-	Summary    *Table
+	Summary    uiTable
 	Views      []StatView
 	ActiveView uiStatView
 }
@@ -144,18 +152,34 @@ func (ctx *TestbedContext) httpMain(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, fmt.Sprintf("%s", err), http.StatusInternalServerError)
 		return
 	}
-
 	uiView := uiStatView{Name: activeView.Name}
 	table, err := activeView.StatsTable()
 	if err != nil {
 		log.Printf("stat table generation failed: %s", err)
 	} else {
-		uiView.Table = table
+		baseColumn := r.FormValue("base_column")
+		if baseColumn != "" {
+			err := table.SetRelativeValues(baseColumn)
+			if err != nil {
+				log.Printf("failed to execute SetRelativeValues: %s", err)
+			}
+		}
+
+		uiView.Table = uiTable{
+			Table: table,
+			Extra: baseColumn != "",
+			ColumnURL: func(column string) string {
+				if column == baseColumn {
+					return ""
+				}
+				return "/?view=" + url.QueryEscape(activeView.Name) + "&base_column=" + url.QueryEscape(column)
+			},
+		}
 	}
 
 	data := &uiMainPage{
 		Name:       ctx.Config.Name,
-		Summary:    ctx.TestbedStatsTable(),
+		Summary:    uiTable{Table: ctx.TestbedStatsTable()},
 		Views:      views,
 		ActiveView: uiView,
 	}
@@ -179,9 +203,14 @@ var mainTemplate = html.CreatePage(`
 <head>
 	<title>{{.Name }} syzkaller</title>
 	{{HEAD}}
-<style>
-
-</style>
+	<style>
+	.positive-delta {
+		color:darkgreen;
+	}
+	.negative-delta {
+		color:darkred;
+	}
+	</style>
 </head>
 <body>
 
@@ -214,20 +243,61 @@ href="?view={{$view.Name}}">█ {{$view.Name}}</a>
 	</table>
 </header>
 
-{{define "Table"}}
-{{if .}}
+{{define "PrintValue"}}
+	{{if or (lt . -100.0) (gt . 100.0)}}
+		{{printf "%.0f" .}}
+	{{else}}
+		{{printf "%.1f" .}}
+	{{end}}
+{{end}}
+
+{{define "PrintExtra"}}
+	{{if .PercentChange}}
+		{{$numVal := (dereference .PercentChange)}}
+		{{if ge $numVal 0.0}}
+			<span class="positive-delta">
+		{{else}}
+			<span class="negative-delta">
+		{{end}}
+		{{printf "%+.1f" $numVal}}%
+		</span>
+	{{end}}
+	{{if .PValue}}
+		p={{printf "%.2f" (dereference .PValue)}}
+	{{end}}
+{{end}}
+
+{{define "PrintTable"}}
+{{$uiTable := .}}
+{{if .Table}}
 <table class="list_table">
 	<tr>
-	<th>{{.TopLeftHeader}}</th>
-	{{range $c := .ColumnHeaders}}
-		<th>{{$c}}</th>
+	<th>{{.Table.TopLeftHeader}}</th>
+	{{range $c := .Table.ColumnHeaders}}
+		<th>
+		{{$url := ""}}
+                {{if $uiTable.ColumnURL}}{{$url = (call $uiTable.ColumnURL $c)}}{{end}}
+			{{if $url}}<a href="{{$url}}">{{$c}}</a>
+			{{else}}
+			{{$c}}
+			{{end}}
+		</th>
+		{{if $uiTable.Extra}}
+		<th>Δ</th>
+		{{end}}
 	{{end}}
 	</tr>
-	{{range $r := .SortedRows}}
+	{{range $r := .Table.SortedRows}}
 	<tr>
 		<td>{{$r}}</td>
-		{{range $c := $.ColumnHeaders}}
-			<td>{{$.Get $r $c}}</td>
+		{{range $c := $uiTable.Table.ColumnHeaders}}
+			{{$cell := ($uiTable.Table.Get $r $c)}}
+			{{if and $cell $uiTable.Extra}}
+			<td>{{template "PrintValue" $cell.Value}}</td>
+			<td>{{template "PrintExtra" $cell}}</td>
+			{{else}}
+			<td>{{$cell}}</td>
+			{{end}}
 		{{end}}
 	</tr>
 	{{end}}
@@ -235,12 +305,12 @@ href="?view={{$view.Name}}">█ {{$view.Name}}</a>
 {{end}}
 {{end}}
 
-{{template "Table" .Summary}}
+{{template "PrintTable" .Summary}}
 
 <b>Stat view "{{$.ActiveView.Name}}"</b><br />
-<a href="/graph?view={{.ActiveView.Name}}&over=fuzzing">Graph over time</a> /
-<a href="/graph?view={{.ActiveView.Name}}&over=exec+total">Graph over executions</a> <br />
-{{template "Table" .ActiveView.Table}}
+<a href="/graph?view={{$.ActiveView.Name}}&over=fuzzing">Graph over time</a> /
+<a href="/graph?view={{$.ActiveView.Name}}&over=exec+total">Graph over executions</a> <br />
+{{template "PrintTable" $.ActiveView.Table}}
 
 </body>
 </html>
diff --git a/tools/syz-testbed/table.go b/tools/syz-testbed/table.go
index 7e293b360..aac065188 100644
--- a/tools/syz-testbed/table.go
+++ b/tools/syz-testbed/table.go
@@ -24,10 +24,10 @@ type Table struct {
 }
 
 type ValueCell struct {
-	Value     float64
-	Sample    *stats.Sample
-	ValueDiff *float64
-	PValue    *float64
+	Value         float64
+	Sample        *stats.Sample
+	PercentChange *float64
+	PValue        *float64
 }
 
 func NewValueCell(sample *stats.Sample) *ValueCell {
@@ -119,3 +119,40 @@ func (t *Table) SaveAsCsv(fileName string) error {
 	defer f.Close()
 	return csv.NewWriter(f).WriteAll(t.ToStrings())
 }
+
+func (t *Table) SetRelativeValues(baseColumn string) error {
+	for rowName, row := range t.Cells {
+		baseCell := t.Get(rowName, baseColumn)
+		if baseCell == nil {
+			return fmt.Errorf("base column %s not found in row %s", baseColumn, rowName)
+		}
+		baseValueCell, ok := baseCell.(*ValueCell)
+		if !ok {
+			return fmt.Errorf("base column cell is not a ValueCell, %T", baseCell)
+		}
+		baseSample := baseValueCell.Sample.RemoveOutliers()
+		for column, cell := range row {
+			if column == baseColumn {
+				continue
+			}
+			valueCell, ok := cell.(*ValueCell)
+			if !ok {
+				continue
+			}
+			if baseValueCell.Value != 0 {
+				valueDiff := valueCell.Value - baseValueCell.Value
+				valueCell.PercentChange = new(float64)
+				*valueCell.PercentChange = valueDiff / baseValueCell.Value * 100
+			}
+
+			cellSample := valueCell.Sample.RemoveOutliers()
+			pval, err := stats.UTest(baseSample, cellSample)
+			if err == nil {
+				// Sometimes it fails because there are too few samples.
+				valueCell.PValue = new(float64)
+				*valueCell.PValue = pval
+			}
+		}
+	}
+	return nil
+}
author	Aleksandr Nogikh <nogikh@google.com>	2021-11-26 16:15:37 +0000
committer	Aleksandr Nogikh <wp32pw@gmail.com>	2021-12-06 14:29:36 +0100
commit	9bc82119f890ecaa556935301dc721f90100d2dd (patch)
tree	6812ecfd10fabb319f3c502cd69b9b7f443d9025 /tools/syz-testbed
parent	8a9bfbcda821c7c38dba195d7ba4a5f34463b49b (diff)