tools/syz-testbed: support experiments with syz-repro

Add a "syz-repro" target and 3 tables:
- List of all performed (and ongoing) reproductions.
- Comparison of repro rate for different bugs on different checkouts.
- Comparison of the share of C reproducers.

6 files changed, 359 insertions, 7 deletions

diff --git a/tools/syz-testbed/checkout.go b/tools/syz-testbed/checkout.go
index 75e79d7a3..e28786930 100644
--- a/tools/syz-testbed/checkout.go
+++ b/tools/syz-testbed/checkout.go
@@ -12,7 +12,10 @@ import (
 	"time"
 
 	syz_instance "github.com/google/syzkaller/pkg/instance"
+	"github.com/google/syzkaller/pkg/mgrconfig"
 	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/pkg/report"
+	"github.com/google/syzkaller/pkg/tool"
 	"github.com/google/syzkaller/pkg/vcs"
 )
 
@@ -24,9 +27,28 @@ type Checkout struct {
 	Running       map[Instance]bool
 	Completed     []RunResult
 	LastRunning   time.Time
+	reporter      *report.Reporter
 	mu            sync.Mutex
 }
 
+func (checkout *Checkout) GetReporter() *report.Reporter {
+	checkout.mu.Lock()
+	defer checkout.mu.Unlock()
+	if checkout.reporter == nil {
+		// Unfortunately, we have no other choice but to parse the config to use our own parser.
+		// TODO: add some tool to syzkaller that would just parse logs and then execute it here?
+		mgrCfg, err := mgrconfig.LoadPartialData(checkout.ManagerConfig)
+		if err != nil {
+			tool.Failf("failed to parse mgr config for %s: %s", checkout.Name, err)
+		}
+		checkout.reporter, err = report.NewReporter(mgrCfg)
+		if err != nil {
+			tool.Failf("failed to get reporter for %s: %s", checkout.Name, err)
+		}
+	}
+	return checkout.reporter
+}
+
 func (checkout *Checkout) AddRunning(instance Instance) {
 	checkout.mu.Lock()
 	defer checkout.mu.Unlock()
diff --git a/tools/syz-testbed/html.go b/tools/syz-testbed/html.go
index f76601a41..25c79db69 100644
--- a/tools/syz-testbed/html.go
+++ b/tools/syz-testbed/html.go
@@ -137,9 +137,13 @@ type uiTable struct {
 }
 
 const (
-	HTMLStatsTable     = "stats"
-	HTMLBugsTable      = "bugs"
-	HTMLBugCountsTable = "bug_counts"
+	HTMLStatsTable         = "stats"
+	HTMLBugsTable          = "bugs"
+	HTMLBugCountsTable     = "bug_counts"
+	HTMLReprosTable        = "repros"
+	HTMLCReprosTable       = "crepros"
+	HTMLReproAttemptsTable = "repro_attempts"
+	HTMLReproDurationTable = "repro_duration"
 )
 
 type uiTableGenerator = func(urlPrefix string, view StatView, r *http.Request) (*uiTable, error)
@@ -170,6 +174,10 @@ func (ctx *TestbedContext) getTableTypes() []uiTableType {
 		{HTMLStatsTable, "Statistics", ctx.httpMainStatsTable},
 		{HTMLBugsTable, "Bugs", ctx.genSimpleTableController((StatView).GenerateBugTable, true)},
 		{HTMLBugCountsTable, "Bug Counts", ctx.genSimpleTableController((StatView).GenerateBugCountsTable, false)},
+		{HTMLReprosTable, "Repros", ctx.genSimpleTableController((StatView).GenerateReproSuccessTable, true)},
+		{HTMLCReprosTable, "C Repros", ctx.genSimpleTableController((StatView).GenerateCReproSuccessTable, true)},
+		{HTMLReproAttemptsTable, "All Repros", ctx.genSimpleTableController((StatView).GenerateReproAttemptsTable, false)},
+		{HTMLReproDurationTable, "Duration", ctx.genSimpleTableController((StatView).GenerateReproDurationTable, true)},
 	}
 	typeList := []uiTableType{}
 	for _, t := range allTypeList {
diff --git a/tools/syz-testbed/instance.go b/tools/syz-testbed/instance.go
index f34df8a98..f3a4c0bc6 100644
--- a/tools/syz-testbed/instance.go
+++ b/tools/syz-testbed/instance.go
@@ -18,6 +18,7 @@ type Instance interface {
 	Run() error
 	Stop()
 	FetchResult() (RunResult, error)
+	Uptime() time.Duration
 }
 
 // The essential information about an active instance.
@@ -26,6 +27,8 @@ type InstanceCommon struct {
 	LogFile         string
 	ExecCommand     string
 	ExecCommandArgs []string
+	StartedAt       time.Time
+	StoppedAt       time.Time
 	stopChannel     chan bool
 }
 
@@ -45,14 +48,15 @@ func (inst *InstanceCommon) Run() error {
 	}
 
 	complete := make(chan error)
+	inst.StartedAt = time.Now()
 	cmd.Start()
 	go func() {
 		complete <- cmd.Wait()
 	}()
 
 	select {
-	case <-complete:
-		return nil
+	case err := <-complete:
+		return err
 	case <-inst.stopChannel:
 		// TODO: handle other OSes?
 		cmd.Process.Signal(os.Interrupt)
@@ -65,8 +69,9 @@ func (inst *InstanceCommon) Run() error {
 			cmd.Process.Kill()
 			<-complete
 		}
-		return nil
 	}
+	inst.StoppedAt = time.Now()
+	return nil
 }
 
 func (inst *InstanceCommon) Stop() {
@@ -76,6 +81,13 @@ func (inst *InstanceCommon) Stop() {
 	}
 }
 
+func (inst *InstanceCommon) Uptime() time.Duration {
+	if !inst.StartedAt.IsZero() && inst.StoppedAt.IsZero() {
+		return time.Since(inst.StartedAt)
+	}
+	return inst.StoppedAt.Sub(inst.StartedAt)
+}
+
 type SyzManagerInstance struct {
 	InstanceCommon
 	SyzkallerInfo
@@ -175,3 +187,56 @@ func (t *SyzManagerTarget) newSyzManagerInstance(slotName, uniqName string, chec
 		RunTime:       t.config.RunTime.Duration,
 	}, nil
 }
+
+type SyzReproInstance struct {
+	InstanceCommon
+	SyzkallerInfo
+	Input      *SyzReproInput
+	ReproFile  string
+	CReproFile string
+	// TODO: we also want to extract source and new bug titles
+}
+
+func (inst *SyzReproInstance) FetchResult() (RunResult, error) {
+	return &SyzReproResult{
+		Input:       inst.Input,
+		ReproFound:  osutil.IsExist(inst.ReproFile),
+		CReproFound: osutil.IsExist(inst.CReproFile),
+		Duration:    inst.Uptime(),
+	}, nil
+}
+
+func (t *SyzReproTarget) newSyzReproInstance(slotName, uniqName string, input *SyzReproInput,
+	checkout *Checkout) (Instance, error) {
+	folder := filepath.Join(checkout.Path, fmt.Sprintf("run-%s", uniqName))
+	common, err := SetupSyzkallerInstance(slotName, folder, checkout)
+	if err != nil {
+		return nil, err
+	}
+
+	reproFile := filepath.Join(folder, "repro.txt")
+	cReproFile := filepath.Join(folder, "crepro.txt")
+	newExecLog := filepath.Join(folder, "execution-log.txt")
+	err = osutil.CopyFile(input.Path, newExecLog)
+	if err != nil {
+		return nil, err
+	}
+	return &SyzReproInstance{
+		InstanceCommon: InstanceCommon{
+			Name:        uniqName,
+			LogFile:     filepath.Join(folder, "log.txt"),
+			ExecCommand: filepath.Join(checkout.Path, "bin", "syz-repro"),
+			ExecCommandArgs: []string{
+				"-config", common.CfgFile,
+				"-output", reproFile,
+				"-crepro", cReproFile,
+				newExecLog,
+			},
+			stopChannel: make(chan bool, 1),
+		},
+		SyzkallerInfo: *common,
+		Input:         input,
+		ReproFile:     reproFile,
+		CReproFile:    cReproFile,
+	}, nil
+}
diff --git a/tools/syz-testbed/stats.go b/tools/syz-testbed/stats.go
index 77fed55b7..5ea3a2931 100644
--- a/tools/syz-testbed/stats.go
+++ b/tools/syz-testbed/stats.go
@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/google/syzkaller/pkg/osutil"
 	"github.com/google/syzkaller/pkg/stats"
@@ -27,6 +28,13 @@ type SyzManagerResult struct {
 	StatRecords []StatRecord
 }
 
+type SyzReproResult struct {
+	Input       *SyzReproInput
+	ReproFound  bool
+	CReproFound bool
+	Duration    time.Duration
+}
+
 // A snapshot of syzkaller statistics at a particular time.
 type StatRecord map[string]uint64
 
@@ -183,6 +191,17 @@ func (group RunResultGroup) SyzManagerResults() []*SyzManagerResult {
 	return ret
 }
 
+func (group RunResultGroup) SyzReproResults() []*SyzReproResult {
+	ret := []*SyzReproResult{}
+	for _, rawRes := range group.Results {
+		res, ok := rawRes.(*SyzReproResult)
+		if ok {
+			ret = append(ret, res)
+		}
+	}
+	return ret
+}
+
 func (group RunResultGroup) AvgStatRecords() []map[string]uint64 {
 	ret := []map[string]uint64{}
 	commonLen := group.minResultLength()
@@ -284,6 +303,99 @@ func (view StatView) InstanceStatsTable() (*Table, error) {
 	return newView.StatsTable()
 }
 
+// How often we find a repro to each crash log.
+func (view StatView) GenerateReproSuccessTable() (*Table, error) {
+	table := NewTable("Bug")
+	for _, group := range view.Groups {
+		table.AddColumn(group.Name)
+	}
+	for _, group := range view.Groups {
+		for _, result := range group.SyzReproResults() {
+			title := result.Input.Title
+			cell, _ := table.Get(title, group.Name).(*RatioCell)
+			if cell == nil {
+				cell = NewRatioCell(0, 0)
+			}
+			cell.TotalCount++
+			if result.ReproFound {
+				cell.TrueCount++
+			}
+			table.Set(title, group.Name, cell)
+		}
+	}
+	return table, nil
+}
+
+// What share of found repros also have a C repro.
+func (view StatView) GenerateCReproSuccessTable() (*Table, error) {
+	table := NewTable("Bug")
+	for _, group := range view.Groups {
+		table.AddColumn(group.Name)
+	}
+	for _, group := range view.Groups {
+		for _, result := range group.SyzReproResults() {
+			if !result.ReproFound {
+				continue
+			}
+			title := result.Input.Title
+			cell, _ := table.Get(title, group.Name).(*RatioCell)
+			if cell == nil {
+				cell = NewRatioCell(0, 0)
+			}
+			cell.TotalCount++
+			if result.CReproFound {
+				cell.TrueCount++
+			}
+			table.Set(title, group.Name, cell)
+		}
+	}
+	return table, nil
+}
+
+// What share of found repros also have a C repro.
+func (view StatView) GenerateReproDurationTable() (*Table, error) {
+	table := NewTable("Bug")
+	for _, group := range view.Groups {
+		table.AddColumn(group.Name)
+	}
+	for _, group := range view.Groups {
+		samples := make(map[string]*stats.Sample)
+		for _, result := range group.SyzReproResults() {
+			title := result.Input.Title
+			var sample *stats.Sample
+			sample, ok := samples[title]
+			if !ok {
+				sample = &stats.Sample{}
+				samples[title] = sample
+			}
+			sample.Xs = append(sample.Xs, result.Duration.Seconds())
+		}
+
+		for title, sample := range samples {
+			table.Set(title, group.Name, NewValueCell(sample))
+		}
+	}
+	return table, nil
+}
+
+// List all repro attempts.
+func (view StatView) GenerateReproAttemptsTable() (*Table, error) {
+	table := NewTable("Result #", "Bug", "Checkout", "Repro found", "C repro found", "Duration")
+	for gid, group := range view.Groups {
+		for rid, result := range group.SyzReproResults() {
+			table.AddRow(
+				fmt.Sprintf("%d-%d", gid, rid),
+				result.Input.Title,
+				group.Name,
+				NewBoolCell(result.ReproFound),
+				NewBoolCell(result.CReproFound),
+				result.Duration.Round(time.Second).String(),
+			)
+		}
+	}
+	return table, nil
+}
+
 // Average bench files of several instances into a single bench file.
 func (group *RunResultGroup) SaveAvgBenchFile(fileName string) error {
 	f, err := os.Create(fileName)
@@ -318,7 +430,7 @@ func (view *StatView) SaveAvgBenches(benchDir string) ([]string, error) {
 
 func (view *StatView) IsEmpty() bool {
 	for _, group := range view.Groups {
-		if group.minResultLength() > 0 {
+		if len(group.Results) > 0 {
 			return false
 		}
 	}
diff --git a/tools/syz-testbed/targets.go b/tools/syz-testbed/targets.go
index c19dec2ca..97941d6ca 100644
--- a/tools/syz-testbed/targets.go
+++ b/tools/syz-testbed/targets.go
@@ -5,11 +5,14 @@ package main
 
 import (
 	"fmt"
+	"io/ioutil"
 	"log"
+	"os"
 	"path/filepath"
 	"sync"
 
 	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/pkg/tool"
 )
 
 // TestbedTarget represents all behavioral differences between specific testbed targets.
@@ -32,6 +35,35 @@ var targetConstructors = map[string]func(cfg *TestbedConfig) TestbedTarget{
 			config: cfg,
 		}
 	},
+	"syz-repro": func(cfg *TestbedConfig) TestbedTarget {
+		inputs := []*SyzReproInput{}
+		if cfg.InputLogs != "" {
+			err := filepath.Walk(cfg.InputLogs, func(path string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				if !info.IsDir() {
+					inputs = append(inputs, &SyzReproInput{
+						Path:  path,
+						runBy: make(map[*Checkout]int),
+					})
+				}
+				return nil
+			})
+			if err != nil {
+				tool.Failf("failed to read logs file directory: %s", err)
+			}
+			// TODO: shuffle?
+		}
+		if len(inputs) == 0 {
+			tool.Failf("no inputs given")
+		}
+		return &SyzReproTarget{
+			config:     cfg,
+			dedupTitle: make(map[string]int),
+			inputs:     inputs,
+		}
+	},
 }
 
 func (t *SyzManagerTarget) NewJob(slotName string, checkouts []*Checkout) (*Checkout, Instance, error) {
@@ -81,3 +113,115 @@ func (t *SyzManagerTarget) SaveStatView(view StatView, dir string) error {
 	_, err = view.SaveAvgBenches(benchDir)
 	return err
 }
+
+// TODO: consider other repro testing modes.
+// E.g. group different logs by title. Then we could also set different sets of inputs
+// for each checkout. It can be important if we improve executor logging.
+type SyzReproTarget struct {
+	config     *TestbedConfig
+	inputs     []*SyzReproInput
+	seqID      int
+	dedupTitle map[string]int
+	mu         sync.Mutex
+}
+
+type SyzReproInput struct {
+	Path      string
+	Title     string
+	Skip      bool
+	origTitle string
+	runBy     map[*Checkout]int
+}
+
+func (inp *SyzReproInput) QueryTitle(checkout *Checkout, dupsMap map[string]int) error {
+	data, err := ioutil.ReadFile(inp.Path)
+	if err != nil {
+		return fmt.Errorf("failed to read: %s", err)
+	}
+	report := checkout.GetReporter().Parse(data)
+	if report == nil {
+		return fmt.Errorf("found no crash")
+	}
+	if inp.Title == "" {
+		inp.origTitle = report.Title
+		inp.Title = report.Title
+		// Some bug titles may be present in multiple log files.
+		// Ensure they are all distict to the user.
+		dupsMap[inp.origTitle]++
+		if dupsMap[inp.Title] > 1 {
+			inp.Title += fmt.Sprintf(" (%d)", dupsMap[inp.origTitle])
+		}
+	}
+	return nil
+}
+
+func (t *SyzReproTarget) NewJob(slotName string, checkouts []*Checkout) (*Checkout, Instance, error) {
+	t.mu.Lock()
+	seqID := t.seqID
+	checkout := checkouts[t.seqID%len(checkouts)]
+	t.seqID++
+	// This may be not the most efficient algorithm, but it guarantees even distribution of
+	// resources and CPU time is negligible in comparison with the amount of time each instance runs.
+	var input *SyzReproInput
+	for _, candidate := range t.inputs {
+		if candidate.Skip {
+			continue
+		}
+		if candidate.runBy[checkout] == 0 {
+			// This is the first time we'll attempt to give this log to the checkout.
+			// Check if it can handle it.
+			err := candidate.QueryTitle(checkout, t.dedupTitle)
+			if err != nil {
+				log.Printf("[log %s]: %s, skipping", candidate.Path, err)
+				candidate.Skip = true
+				continue
+			}
+		}
+		if input == nil || input.runBy[checkout] > candidate.runBy[checkout] {
+			// Pick the least executed one.
+			input = candidate
+		}
+	}
+
+	if input == nil {
+		t.mu.Unlock()
+		return nil, nil, fmt.Errorf("no available inputs")
+	}
+	input.runBy[checkout]++
+	t.mu.Unlock()
+
+	uniqName := fmt.Sprintf("%s-%d", checkout.Name, seqID)
+	instance, err := t.newSyzReproInstance(slotName, uniqName, input, checkout)
+	if err != nil {
+		return nil, nil, err
+	}
+	return checkout, instance, nil
+}
+
+func (t *SyzReproTarget) SupportsHTMLView(key string) bool {
+	supported := map[string]bool{
+		HTMLReprosTable:        true,
+		HTMLCReprosTable:       true,
+		HTMLReproAttemptsTable: true,
+		HTMLReproDurationTable: true,
+	}
+	return supported[key]
+}
+
+func (t *SyzReproTarget) SaveStatView(view StatView, dir string) error {
+	tableStats := map[string]func(view StatView) (*Table, error){
+		"repro_success.csv":   (StatView).GenerateReproSuccessTable,
+		"crepros_success.csv": (StatView).GenerateCReproSuccessTable,
+		"repro_attempts.csv":  (StatView).GenerateReproAttemptsTable,
+		"repro_duration.csv":  (StatView).GenerateReproDurationTable,
+	}
+	for fileName, genFunc := range tableStats {
+		table, err := genFunc(view)
+		if err == nil {
+			table.SaveAsCsv(filepath.Join(dir, fileName))
+		} else {
+			log.Printf("stat generation error: %s", err)
+		}
+	}
+	return nil
+}
diff --git a/tools/syz-testbed/testbed.go b/tools/syz-testbed/testbed.go
index 6f7a1868d..0af48bc1d 100644
--- a/tools/syz-testbed/testbed.go
+++ b/tools/syz-testbed/testbed.go
@@ -33,6 +33,7 @@ type TestbedConfig struct {
 	Target        string           `json:"target"`         // what application to test
 	MaxInstances  int              `json:"max_instances"`  // max # of simultaneously running instances
 	RunTime       DurationConfig   `json:"run_time"`       // lifetime of an instance (default "24h")
+	InputLogs     string           `json:"input_logs"`     // folder with logs to be processed by syz-repro
 	HTTP          string           `json:"http"`           // on which port to set up a simple web dashboard
 	BenchCmp      string           `json:"benchcmp"`       // path to the syz-benchcmp executable
 	Corpus        string           `json:"corpus"`         // path to the corpus file