tools/syz-benchcmp: add utility for visualization of syz-manager benchmarking results

1 files changed, 247 insertions, 0 deletions

diff --git a/tools/syz-benchcmp/benchcmp.go b/tools/syz-benchcmp/benchcmp.go
new file mode 100644
index 000000000..bdfed9e6e
--- /dev/null
+++ b/tools/syz-benchcmp/benchcmp.go
@@ -0,0 +1,247 @@
+// Copyright 2017 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+// syz-benchcmp visualizes syz-manager benchmarking results.
+// First, run syz-manager with -bench=old flag.
+// Then, do experimental modifications and run syz-manager again with -bench=new flag.
+// Then, run syz-benchcmp old new.
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"html/template"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+)
+
+var (
+	flagSkip = flag.Int("skip", -30, "skip that many seconds after start (skip first 20% by default)")
+)
+
+type Graph struct {
+	Name    string
+	Headers []string
+	Points  []Point
+}
+
+type Point struct {
+	Time uint64
+	Vals []uint64
+}
+
+func main() {
+	flag.Parse()
+	if len(flag.Args()) == 0 {
+		fmt.Fprintf(os.Stderr, "usage: syz-benchcmp [flags] bench_file0 [bench_file1 [bench_file2]]...\n")
+		flag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	graphs := []*Graph{
+		&Graph{Name: "coverage"},
+		&Graph{Name: "corpus"},
+		&Graph{Name: "exec total"},
+		&Graph{Name: "crash types"},
+	}
+	for i, fname := range flag.Args() {
+		data := readFile(fname)
+		for _, g := range graphs {
+			g.Headers = append(g.Headers, filepath.Base(fname))
+			for _, v := range data {
+				pt := Point{
+					Time: v["fuzzing"],
+					Vals: make([]uint64, len(flag.Args())),
+				}
+				pt.Vals[i] = v[g.Name]
+				g.Points = append(g.Points, pt)
+			}
+		}
+	}
+	for _, g := range graphs {
+		if len(g.Points) == 0 {
+			failf("no data points")
+		}
+		sort.Sort(pointSlice(g.Points))
+		skipStart(g)
+		restoreMissingPoints(g)
+	}
+	printFinalStats(graphs)
+	display(graphs)
+}
+
+func readFile(fname string) (data []map[string]uint64) {
+	f, err := os.Open(fname)
+	if err != nil {
+		failf("failed to open input file: %v", err)
+	}
+	defer f.Close()
+	dec := json.NewDecoder(bufio.NewReader(f))
+	for dec.More() {
+		v := make(map[string]uint64)
+		if err := dec.Decode(&v); err != nil {
+			failf("failed to decode input file %v: %v", fname, err)
+		}
+		data = append(data, v)
+	}
+	return
+}
+
+func skipStart(g *Graph) {
+	skipTime := uint64(*flagSkip)
+	if *flagSkip < 0 {
+		// Negative skip means percents.
+		max := g.Points[len(g.Points)-1].Time
+		skipTime = max * -skipTime / 100
+	}
+	if skipTime > 0 {
+		skip := sort.Search(len(g.Points), func(i int) bool {
+			return g.Points[i].Time > skipTime
+		})
+		g.Points = g.Points[skip:]
+	}
+}
+
+func restoreMissingPoints(g *Graph) {
+	for i := range g.Headers {
+		// Find previous and next non-zero point for each zero point,
+		// and restore its value with linear inerpolation.
+		type Pt struct {
+			Time uint64
+			Val  uint64
+		}
+		var prev Pt
+		prevs := make(map[uint64]Pt)
+		for _, pt := range g.Points {
+			if pt.Vals[i] != 0 {
+				prev = Pt{pt.Time, pt.Vals[i]}
+				continue
+			}
+			prevs[pt.Time] = prev
+		}
+		var next Pt
+		for pti := len(g.Points) - 1; pti >= 0; pti-- {
+			pt := g.Points[pti]
+			if pt.Vals[i] != 0 {
+				next = Pt{pt.Time, pt.Vals[i]}
+				continue
+			}
+			prev := prevs[pt.Time]
+			if prev.Val == 0 || next.Val == 0 {
+				continue
+			}
+			pt.Vals[i] = prev.Val
+			if next.Time != prev.Time {
+				// Use signed calculations as corpus can go backwards.
+				pt.Vals[i] += uint64(int64(next.Val-prev.Val) * int64(pt.Time-prev.Time) / int64(next.Time-prev.Time))
+			}
+		}
+	}
+}
+
+func printFinalStats(graphs []*Graph) {
+	for i := 1; i < len(graphs[0].Headers); i++ {
+		fmt.Printf("%-12v%16v%16v%16v\n", "", graphs[0].Headers[0], graphs[0].Headers[i], "diff")
+		for _, g := range graphs {
+			lastNonZero := func(x int) uint64 {
+				for j := len(g.Points) - 1; j >= 0; j-- {
+					if v := g.Points[j].Vals[x]; v != 0 {
+						return v
+					}
+				}
+				return 0
+			}
+			old := lastNonZero(0)
+			new := lastNonZero(i)
+			fmt.Printf("%-12v%16v%16v%+16d\n", g.Name, old, new, int64(new-old))
+		}
+		fmt.Printf("\n")
+	}
+}
+
+func display(graphs []*Graph) {
+	outf, err := ioutil.TempFile("", "")
+	if err != nil {
+		failf("failed to create temp file: %v", err)
+	}
+	if err := htmlTemplate.Execute(outf, graphs); err != nil {
+		failf("failed to execute template: %v", err)
+	}
+	outf.Close()
+	name := outf.Name() + ".html"
+	if err := os.Rename(outf.Name(), name); err != nil {
+		failf("failed to rename file: %v", err)
+	}
+	if err := exec.Command("xdg-open", name).Start(); err != nil {
+		failf("failed to start browser: %v", err)
+	}
+}
+
+type pointSlice []Point
+
+func (a pointSlice) Len() int           { return len(a) }
+func (a pointSlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a pointSlice) Less(i, j int) bool { return a[i].Time < a[j].Time }
+
+func failf(msg string, args ...interface{}) {
+	fmt.Fprintf(os.Stderr, msg+"\n", args...)
+	os.Exit(1)
+}
+
+var htmlTemplate = template.Must(
+	template.New("").Parse(`
+<!doctype html>
+<html>
+  <head>
+    <title>Syzkaller Bench</title>
+    <script type="text/javascript" src="https://www.google.com/jsapi"></script>
+    <script type="text/javascript">
+      google.load("visualization", "1", {packages:["corechart"]});
+      google.setOnLoadCallback(drawCharts);
+      function drawCharts() {
+        {{range $id, $graph := .}}
+        {
+          var data = new google.visualization.DataTable();
+          data.addColumn({type: 'number'});
+          {{range $graph.Headers}}
+            data.addColumn({type: 'number', label: '{{.}}'});
+          {{end}}
+          data.addRows([
+            {{range $graph.Points}} [ {{.Time}}, {{range .Vals}} {{if .}} {{.}} {{end}}, {{end}}
+          ],
+          {{end}}
+          ]);
+          new google.visualization.LineChart(document.getElementById('graph_div_{{$id}}')).
+            draw(data, {
+              title: '{{$graph.Name}}',
+              width: "100%",
+              height: document.documentElement.clientHeight * 0.48,
+              legend: {position: "in"},
+              focusTarget: "category",
+              hAxis: {title: "Time, sec"},
+              chartArea: {left: "5%", top: "5%", width: "90%", height:"85%"}
+            })
+        }
+        {{end}}
+      }
+    </script>
+</head>
+<body>
+  <table style="width: 100%; height: 98vh">
+    <tr>
+      <td style="width: 50%"> <div id="graph_div_0"></div> </td>
+      <td style="width: 50%"> <div id="graph_div_2"></div> </td>
+    </tr>
+    <tr>
+      <td style="width: 50%"> <div id="graph_div_1"></div> </td>
+      <td style="width: 50%"> <div id="graph_div_3"></div> </td>
+    </tr>
+  </table>
+</body>
+</html>
+`))