tools/syz-testbed: introduce a special type for table

This simplifies table generation and will let us more easily implement
relative difference and p-value calculation and printing.

4 files changed, 150 insertions, 63 deletions

diff --git a/tools/syz-testbed/html.go b/tools/syz-testbed/html.go
index fa3786cd4..196c12a3c 100644
--- a/tools/syz-testbed/html.go
+++ b/tools/syz-testbed/html.go
@@ -12,7 +12,6 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"sort"
 	"time"
 
 	"github.com/google/syzkaller/pkg/html"
@@ -103,12 +102,12 @@ func (ctx *TestbedContext) httpGraph(w http.ResponseWriter, r *http.Request) {
 
 type uiStatView struct {
 	Name  string
-	Table [][]string
+	Table *Table
 }
 
 type uiMainPage struct {
 	Name    string
-	Summary [][]string
+	Summary *Table
 	Views   []uiStatView
 }
 
@@ -125,12 +124,6 @@ func (ctx *TestbedContext) httpMain(w http.ResponseWriter, r *http.Request) {
 			log.Printf("stat table generation failed: %s", err)
 			continue
 		}
-		sort.SliceStable(table, func(i, j int) bool {
-			if len(table[i]) == 0 || len(table[j]) == 0 {
-				return i < j
-			}
-			return table[i][0] < table[j][0]
-		})
 		uiViews = append(uiViews, uiStatView{
 			Name:  view.Name,
 			Table: table,
@@ -167,12 +160,19 @@ var mainTemplate = html.CreatePage(`
 {{define "Table"}}
 {{if .}}
 <table class="list_table">
-	{{range $c := .}}
 	<tr>
-	{{range $v := $c}}
-		<td>{{$v}}</td>
+	<th>{{.TopLeftHeader}}</th>
+	{{range $c := .ColumnHeaders}}
+		<th>{{$c}}</th>
 	{{end}}
 	</tr>
+	{{range $r := .SortedRows}}
+	<tr>
+		<td>{{$r}}</td>
+		{{range $c := $.ColumnHeaders}}
+			<td>{{$.Get $r $c}}</td>
+		{{end}}
+	</tr>
 	{{end}}
 </table>
 {{end}}
diff --git a/tools/syz-testbed/stats.go b/tools/syz-testbed/stats.go
index bd3c821c9..3086e6741 100644
--- a/tools/syz-testbed/stats.go
+++ b/tools/syz-testbed/stats.go
@@ -4,7 +4,6 @@
 package main
 
 import (
-	"encoding/csv"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -128,28 +127,21 @@ func summarizeBugs(groups []RunResultGroup) ([]*BugSummary, error) {
 
 // For each checkout, take the union of sets of bugs found by each instance.
 // Then output these unions as a single table.
-func (view StatView) GenerateBugTable() ([][]string, error) {
-	table := [][]string{}
-	titles := []string{""}
+func (view StatView) GenerateBugTable() (*Table, error) {
+	table := NewTable("Bug")
 	for _, group := range view.Groups {
-		titles = append(titles, group.Name)
+		table.AddColumn(group.Name)
 	}
 	summaries, err := summarizeBugs(view.Groups)
 	if err != nil {
 		return nil, err
 	}
-
-	table = append(table, titles)
 	for _, bug := range summaries {
-		row := []string{bug.title}
 		for _, group := range view.Groups {
-			val := ""
 			if bug.found[group.Name] {
-				val = "YES"
+				table.Set(bug.title, group.Name, "YES")
 			}
-			row = append(row, val)
 		}
-		table = append(table, row)
 	}
 	return table, nil
 }
@@ -189,7 +181,7 @@ func (group RunResultGroup) groupNthRecord(i int) map[string]*stats.Sample {
 	return groupSamples(records)
 }
 
-func (view StatView) StatsTable() ([][]string, error) {
+func (view StatView) StatsTable() (*Table, error) {
 	commonLen := 0
 	for _, group := range view.Groups {
 		minLen := group.minResultLength()
@@ -200,12 +192,11 @@ func (view StatView) StatsTable() ([][]string, error) {
 			commonLen = minLen
 		}
 	}
-	if commonLen == 0 {
-		return nil, fmt.Errorf("not enough stat records")
-	}
 	// Map: stats key x group name -> value.
+	table := NewTable("Property")
 	cells := make(map[string]map[string]string)
 	for _, group := range view.Groups {
+		table.AddColumn(group.Name)
 		if group.minResultLength() == 0 {
 			// Skip empty groups.
 			continue
@@ -215,25 +206,13 @@ func (view StatView) StatsTable() ([][]string, error) {
 			if _, ok := cells[key]; !ok {
 				cells[key] = make(map[string]string)
 			}
-			cells[key][group.Name] = fmt.Sprintf("%d", int64(sample.Median()))
-		}
-	}
-	title := []string{""}
-	for _, group := range view.Groups {
-		title = append(title, group.Name)
-	}
-	table := [][]string{title}
-	for key, valuesMap := range cells {
-		row := []string{key}
-		for _, group := range view.Groups {
-			row = append(row, valuesMap[group.Name])
+			table.Set(key, group.Name, NewValueCell(sample))
 		}
-		table = append(table, row)
 	}
 	return table, nil
 }
 
-func (view StatView) InstanceStatsTable() ([][]string, error) {
+func (view StatView) InstanceStatsTable() (*Table, error) {
 	newView := StatView{}
 	for _, group := range view.Groups {
 		for i, result := range group.Results {
@@ -277,12 +256,3 @@ func (view *StatView) SaveAvgBenches(benchDir string) ([]string, error) {
 	}
 	return files, nil
 }
-
-func SaveTableAsCsv(table [][]string, fileName string) error {
-	f, err := os.Create(fileName)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	return csv.NewWriter(f).WriteAll(table)
-}
diff --git a/tools/syz-testbed/table.go b/tools/syz-testbed/table.go
new file mode 100644
index 000000000..7e293b360
--- /dev/null
+++ b/tools/syz-testbed/table.go
@@ -0,0 +1,121 @@
+// Copyright 2021 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"encoding/csv"
+	"fmt"
+	"math"
+	"os"
+	"sort"
+
+	"github.com/google/syzkaller/pkg/stats"
+)
+
+type Cell = interface{}
+
+// All tables that syz-testbed generates have named columns and rows.
+// Table type simplifies generation and processing of such tables.
+type Table struct {
+	TopLeftHeader string
+	ColumnHeaders []string
+	Cells         map[string]map[string]Cell
+}
+
+type ValueCell struct {
+	Value     float64
+	Sample    *stats.Sample
+	ValueDiff *float64
+	PValue    *float64
+}
+
+func NewValueCell(sample *stats.Sample) *ValueCell {
+	return &ValueCell{Value: sample.Median(), Sample: sample}
+}
+
+func (c *ValueCell) String() string {
+	const fractionCutoff = 100
+	if math.Abs(c.Value) < fractionCutoff {
+		return fmt.Sprintf("%.1f", c.Value)
+	}
+	return fmt.Sprintf("%.0f", math.Round(c.Value))
+}
+
+func NewTable(topLeft string, columns ...string) *Table {
+	return &Table{
+		TopLeftHeader: topLeft,
+		ColumnHeaders: columns,
+	}
+}
+
+func (t *Table) Get(row, column string) Cell {
+	if t.Cells == nil {
+		return nil
+	}
+	rowMap := t.Cells[row]
+	if rowMap == nil {
+		return nil
+	}
+	return rowMap[column]
+}
+
+func (t *Table) Set(row, column string, value Cell) {
+	if t.Cells == nil {
+		t.Cells = make(map[string]map[string]Cell)
+	}
+	rowMap, ok := t.Cells[row]
+	if !ok {
+		rowMap = make(map[string]Cell)
+		t.Cells[row] = rowMap
+	}
+	rowMap[column] = value
+}
+
+func (t *Table) AddColumn(column string) {
+	t.ColumnHeaders = append(t.ColumnHeaders, column)
+}
+
+func (t *Table) AddRow(row string, cells ...Cell) {
+	if len(cells) != len(t.ColumnHeaders) {
+		panic("AddRow: the length of the row does not equal the number of columns")
+	}
+	for i, col := range t.ColumnHeaders {
+		t.Set(row, col, cells[i])
+	}
+}
+
+func (t *Table) SortedRows() []string {
+	rows := []string{}
+	for key := range t.Cells {
+		rows = append(rows, key)
+	}
+	sort.Strings(rows)
+	return rows
+}
+
+func (t *Table) ToStrings() [][]string {
+	table := [][]string{}
+	headers := append([]string{t.TopLeftHeader}, t.ColumnHeaders...)
+	table = append(table, headers)
+	if t.Cells != nil {
+		rowHeaders := t.SortedRows()
+		for _, row := range rowHeaders {
+			tableRow := []string{row}
+			for _, column := range t.ColumnHeaders {
+				tableRow = append(tableRow, fmt.Sprintf("%s", t.Get(row, column)))
+			}
+			table = append(table, tableRow)
+		}
+	}
+	return table
+}
+
+func (t *Table) SaveAsCsv(fileName string) error {
+	f, err := os.Create(fileName)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return csv.NewWriter(f).WriteAll(t.ToStrings())
+}
diff --git a/tools/syz-testbed/testbed.go b/tools/syz-testbed/testbed.go
index dee9ffc73..192784763 100644
--- a/tools/syz-testbed/testbed.go
+++ b/tools/syz-testbed/testbed.go
@@ -163,8 +163,7 @@ func (ctx *TestbedContext) saveStatView(view StatView) error {
 	if err != nil {
 		return fmt.Errorf("failed to create %s: %s", benchDir, err)
 	}
-
-	tableStats := map[string]func(view StatView) ([][]string, error){
+	tableStats := map[string]func(view StatView) (*Table, error){
 		"bugs.csv":           (StatView).GenerateBugTable,
 		"checkout_stats.csv": (StatView).StatsTable,
 		"instance_stats.csv": (StatView).InstanceStatsTable,
@@ -172,30 +171,27 @@ func (ctx *TestbedContext) saveStatView(view StatView) error {
 	for fileName, genFunc := range tableStats {
 		table, err := genFunc(view)
 		if err == nil {
-			SaveTableAsCsv(table, filepath.Join(dir, fileName))
+			table.SaveAsCsv(filepath.Join(dir, fileName))
 		} else {
-			log.Printf("some error: %s", err)
+			log.Printf("stat generation error: %s", err)
 		}
 	}
 	_, err = view.SaveAvgBenches(benchDir)
 	return err
 }
 
-func (ctx *TestbedContext) TestbedStatsTable() [][]string {
-	table := [][]string{
-		{"Checkout", "Running", "Completed", "Until reset"},
-	}
+func (ctx *TestbedContext) TestbedStatsTable() *Table {
+	table := NewTable("Checkout", "Running", "Completed", "Until reset")
 	for _, checkout := range ctx.Checkouts {
 		until := "-"
 		if ctx.NextRestart.After(time.Now()) {
 			until = time.Until(ctx.NextRestart).Round(time.Second).String()
 		}
-		table = append(table, []string{
-			checkout.Name,
+		table.AddRow(checkout.Name,
 			fmt.Sprintf("%d", len(checkout.Running)),
 			fmt.Sprintf("%d", len(checkout.Completed)),
 			until,
-		})
+		)
 	}
 	return table
 }
@@ -215,7 +211,7 @@ func (ctx *TestbedContext) SaveStats() error {
 		}
 	}
 	table := ctx.TestbedStatsTable()
-	return SaveTableAsCsv(table, filepath.Join(ctx.Config.Workdir, "testbed.csv"))
+	return table.SaveAsCsv(filepath.Join(ctx.Config.Workdir, "testbed.csv"))
 }
 
 func (ctx *TestbedContext) generateInstances(count int) ([]*Instance, error) {