From 9fca97cadd2e63056f40e69d80ca1e798b2a18e8 Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Thu, 25 Nov 2021 17:55:58 +0000 Subject: tools/syz-testbed: introduce a special type for table This simplifies table generation and will let us more easily implement relative difference and p-value calculation and printing. --- tools/syz-testbed/html.go | 24 ++++----- tools/syz-testbed/stats.go | 48 ++++------------- tools/syz-testbed/table.go | 121 +++++++++++++++++++++++++++++++++++++++++++ tools/syz-testbed/testbed.go | 20 +++---- 4 files changed, 150 insertions(+), 63 deletions(-) create mode 100644 tools/syz-testbed/table.go (limited to 'tools') diff --git a/tools/syz-testbed/html.go b/tools/syz-testbed/html.go index fa3786cd4..196c12a3c 100644 --- a/tools/syz-testbed/html.go +++ b/tools/syz-testbed/html.go @@ -12,7 +12,6 @@ import ( "net" "net/http" "os" - "sort" "time" "github.com/google/syzkaller/pkg/html" @@ -103,12 +102,12 @@ func (ctx *TestbedContext) httpGraph(w http.ResponseWriter, r *http.Request) { type uiStatView struct { Name string - Table [][]string + Table *Table } type uiMainPage struct { Name string - Summary [][]string + Summary *Table Views []uiStatView } @@ -125,12 +124,6 @@ func (ctx *TestbedContext) httpMain(w http.ResponseWriter, r *http.Request) { log.Printf("stat table generation failed: %s", err) continue } - sort.SliceStable(table, func(i, j int) bool { - if len(table[i]) == 0 || len(table[j]) == 0 { - return i < j - } - return table[i][0] < table[j][0] - }) uiViews = append(uiViews, uiStatView{ Name: view.Name, Table: table, @@ -167,12 +160,19 @@ var mainTemplate = html.CreatePage(` {{define "Table"}} {{if .}} - {{range $c := .}} - {{range $v := $c}} - + + {{range $c := .ColumnHeaders}} + {{end}} + {{range $r := .SortedRows}} + + + {{range $c := $.ColumnHeaders}} + + {{end}} + {{end}}
{{$v}}{{.TopLeftHeader}}{{$c}}
{{$r}}{{$.Get $r $c}}
{{end}} diff --git a/tools/syz-testbed/stats.go b/tools/syz-testbed/stats.go index bd3c821c9..3086e6741 100644 --- a/tools/syz-testbed/stats.go +++ b/tools/syz-testbed/stats.go @@ -4,7 +4,6 @@ package main import ( - "encoding/csv" "encoding/json" "fmt" "io/ioutil" @@ -128,28 +127,21 @@ func summarizeBugs(groups []RunResultGroup) ([]*BugSummary, error) { // For each checkout, take the union of sets of bugs found by each instance. // Then output these unions as a single table. -func (view StatView) GenerateBugTable() ([][]string, error) { - table := [][]string{} - titles := []string{""} +func (view StatView) GenerateBugTable() (*Table, error) { + table := NewTable("Bug") for _, group := range view.Groups { - titles = append(titles, group.Name) + table.AddColumn(group.Name) } summaries, err := summarizeBugs(view.Groups) if err != nil { return nil, err } - - table = append(table, titles) for _, bug := range summaries { - row := []string{bug.title} for _, group := range view.Groups { - val := "" if bug.found[group.Name] { - val = "YES" + table.Set(bug.title, group.Name, "YES") } - row = append(row, val) } - table = append(table, row) } return table, nil } @@ -189,7 +181,7 @@ func (group RunResultGroup) groupNthRecord(i int) map[string]*stats.Sample { return groupSamples(records) } -func (view StatView) StatsTable() ([][]string, error) { +func (view StatView) StatsTable() (*Table, error) { commonLen := 0 for _, group := range view.Groups { minLen := group.minResultLength() @@ -200,12 +192,11 @@ func (view StatView) StatsTable() ([][]string, error) { commonLen = minLen } } - if commonLen == 0 { - return nil, fmt.Errorf("not enough stat records") - } // Map: stats key x group name -> value. + table := NewTable("Property") cells := make(map[string]map[string]string) for _, group := range view.Groups { + table.AddColumn(group.Name) if group.minResultLength() == 0 { // Skip empty groups. continue @@ -215,25 +206,13 @@ func (view StatView) StatsTable() ([][]string, error) { if _, ok := cells[key]; !ok { cells[key] = make(map[string]string) } - cells[key][group.Name] = fmt.Sprintf("%d", int64(sample.Median())) - } - } - title := []string{""} - for _, group := range view.Groups { - title = append(title, group.Name) - } - table := [][]string{title} - for key, valuesMap := range cells { - row := []string{key} - for _, group := range view.Groups { - row = append(row, valuesMap[group.Name]) + table.Set(key, group.Name, NewValueCell(sample)) } - table = append(table, row) } return table, nil } -func (view StatView) InstanceStatsTable() ([][]string, error) { +func (view StatView) InstanceStatsTable() (*Table, error) { newView := StatView{} for _, group := range view.Groups { for i, result := range group.Results { @@ -277,12 +256,3 @@ func (view *StatView) SaveAvgBenches(benchDir string) ([]string, error) { } return files, nil } - -func SaveTableAsCsv(table [][]string, fileName string) error { - f, err := os.Create(fileName) - if err != nil { - return err - } - defer f.Close() - return csv.NewWriter(f).WriteAll(table) -} diff --git a/tools/syz-testbed/table.go b/tools/syz-testbed/table.go new file mode 100644 index 000000000..7e293b360 --- /dev/null +++ b/tools/syz-testbed/table.go @@ -0,0 +1,121 @@ +// Copyright 2021 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "encoding/csv" + "fmt" + "math" + "os" + "sort" + + "github.com/google/syzkaller/pkg/stats" +) + +type Cell = interface{} + +// All tables that syz-testbed generates have named columns and rows. +// Table type simplifies generation and processing of such tables. +type Table struct { + TopLeftHeader string + ColumnHeaders []string + Cells map[string]map[string]Cell +} + +type ValueCell struct { + Value float64 + Sample *stats.Sample + ValueDiff *float64 + PValue *float64 +} + +func NewValueCell(sample *stats.Sample) *ValueCell { + return &ValueCell{Value: sample.Median(), Sample: sample} +} + +func (c *ValueCell) String() string { + const fractionCutoff = 100 + if math.Abs(c.Value) < fractionCutoff { + return fmt.Sprintf("%.1f", c.Value) + } + return fmt.Sprintf("%.0f", math.Round(c.Value)) +} + +func NewTable(topLeft string, columns ...string) *Table { + return &Table{ + TopLeftHeader: topLeft, + ColumnHeaders: columns, + } +} + +func (t *Table) Get(row, column string) Cell { + if t.Cells == nil { + return nil + } + rowMap := t.Cells[row] + if rowMap == nil { + return nil + } + return rowMap[column] +} + +func (t *Table) Set(row, column string, value Cell) { + if t.Cells == nil { + t.Cells = make(map[string]map[string]Cell) + } + rowMap, ok := t.Cells[row] + if !ok { + rowMap = make(map[string]Cell) + t.Cells[row] = rowMap + } + rowMap[column] = value +} + +func (t *Table) AddColumn(column string) { + t.ColumnHeaders = append(t.ColumnHeaders, column) +} + +func (t *Table) AddRow(row string, cells ...Cell) { + if len(cells) != len(t.ColumnHeaders) { + panic("AddRow: the length of the row does not equal the number of columns") + } + for i, col := range t.ColumnHeaders { + t.Set(row, col, cells[i]) + } +} + +func (t *Table) SortedRows() []string { + rows := []string{} + for key := range t.Cells { + rows = append(rows, key) + } + sort.Strings(rows) + return rows +} + +func (t *Table) ToStrings() [][]string { + table := [][]string{} + headers := append([]string{t.TopLeftHeader}, t.ColumnHeaders...) + table = append(table, headers) + if t.Cells != nil { + rowHeaders := t.SortedRows() + for _, row := range rowHeaders { + tableRow := []string{row} + for _, column := range t.ColumnHeaders { + tableRow = append(tableRow, fmt.Sprintf("%s", t.Get(row, column))) + } + table = append(table, tableRow) + } + } + return table +} + +func (t *Table) SaveAsCsv(fileName string) error { + f, err := os.Create(fileName) + if err != nil { + return err + } + defer f.Close() + return csv.NewWriter(f).WriteAll(t.ToStrings()) +} diff --git a/tools/syz-testbed/testbed.go b/tools/syz-testbed/testbed.go index dee9ffc73..192784763 100644 --- a/tools/syz-testbed/testbed.go +++ b/tools/syz-testbed/testbed.go @@ -163,8 +163,7 @@ func (ctx *TestbedContext) saveStatView(view StatView) error { if err != nil { return fmt.Errorf("failed to create %s: %s", benchDir, err) } - - tableStats := map[string]func(view StatView) ([][]string, error){ + tableStats := map[string]func(view StatView) (*Table, error){ "bugs.csv": (StatView).GenerateBugTable, "checkout_stats.csv": (StatView).StatsTable, "instance_stats.csv": (StatView).InstanceStatsTable, @@ -172,30 +171,27 @@ func (ctx *TestbedContext) saveStatView(view StatView) error { for fileName, genFunc := range tableStats { table, err := genFunc(view) if err == nil { - SaveTableAsCsv(table, filepath.Join(dir, fileName)) + table.SaveAsCsv(filepath.Join(dir, fileName)) } else { - log.Printf("some error: %s", err) + log.Printf("stat generation error: %s", err) } } _, err = view.SaveAvgBenches(benchDir) return err } -func (ctx *TestbedContext) TestbedStatsTable() [][]string { - table := [][]string{ - {"Checkout", "Running", "Completed", "Until reset"}, - } +func (ctx *TestbedContext) TestbedStatsTable() *Table { + table := NewTable("Checkout", "Running", "Completed", "Until reset") for _, checkout := range ctx.Checkouts { until := "-" if ctx.NextRestart.After(time.Now()) { until = time.Until(ctx.NextRestart).Round(time.Second).String() } - table = append(table, []string{ - checkout.Name, + table.AddRow(checkout.Name, fmt.Sprintf("%d", len(checkout.Running)), fmt.Sprintf("%d", len(checkout.Completed)), until, - }) + ) } return table } @@ -215,7 +211,7 @@ func (ctx *TestbedContext) SaveStats() error { } } table := ctx.TestbedStatsTable() - return SaveTableAsCsv(table, filepath.Join(ctx.Config.Workdir, "testbed.csv")) + return table.SaveAsCsv(filepath.Join(ctx.Config.Workdir, "testbed.csv")) } func (ctx *TestbedContext) generateInstances(count int) ([]*Instance, error) { -- cgit mrf-deployment