From aba4330c5c9f0ef53b81382508da7ded88ddffc0 Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Fri, 11 Feb 2022 10:04:55 +0000 Subject: tools/syz-testbed: support experiments with syz-repro Add a "syz-repro" target and 3 tables: - List of all performed (and ongoing) reproductions. - Comparison of repro rate for different bugs on different checkouts. - Comparison of the share of C reproducers. --- tools/syz-testbed/checkout.go | 22 +++++++ tools/syz-testbed/html.go | 14 +++- tools/syz-testbed/instance.go | 71 ++++++++++++++++++++- tools/syz-testbed/stats.go | 114 ++++++++++++++++++++++++++++++++- tools/syz-testbed/targets.go | 144 ++++++++++++++++++++++++++++++++++++++++++ tools/syz-testbed/testbed.go | 1 + 6 files changed, 359 insertions(+), 7 deletions(-) (limited to 'tools/syz-testbed') diff --git a/tools/syz-testbed/checkout.go b/tools/syz-testbed/checkout.go index 75e79d7a3..e28786930 100644 --- a/tools/syz-testbed/checkout.go +++ b/tools/syz-testbed/checkout.go @@ -12,7 +12,10 @@ import ( "time" syz_instance "github.com/google/syzkaller/pkg/instance" + "github.com/google/syzkaller/pkg/mgrconfig" "github.com/google/syzkaller/pkg/osutil" + "github.com/google/syzkaller/pkg/report" + "github.com/google/syzkaller/pkg/tool" "github.com/google/syzkaller/pkg/vcs" ) @@ -24,9 +27,28 @@ type Checkout struct { Running map[Instance]bool Completed []RunResult LastRunning time.Time + reporter *report.Reporter mu sync.Mutex } +func (checkout *Checkout) GetReporter() *report.Reporter { + checkout.mu.Lock() + defer checkout.mu.Unlock() + if checkout.reporter == nil { + // Unfortunately, we have no other choice but to parse the config to use our own parser. + // TODO: add some tool to syzkaller that would just parse logs and then execute it here? + mgrCfg, err := mgrconfig.LoadPartialData(checkout.ManagerConfig) + if err != nil { + tool.Failf("failed to parse mgr config for %s: %s", checkout.Name, err) + } + checkout.reporter, err = report.NewReporter(mgrCfg) + if err != nil { + tool.Failf("failed to get reporter for %s: %s", checkout.Name, err) + } + } + return checkout.reporter +} + func (checkout *Checkout) AddRunning(instance Instance) { checkout.mu.Lock() defer checkout.mu.Unlock() diff --git a/tools/syz-testbed/html.go b/tools/syz-testbed/html.go index f76601a41..25c79db69 100644 --- a/tools/syz-testbed/html.go +++ b/tools/syz-testbed/html.go @@ -137,9 +137,13 @@ type uiTable struct { } const ( - HTMLStatsTable = "stats" - HTMLBugsTable = "bugs" - HTMLBugCountsTable = "bug_counts" + HTMLStatsTable = "stats" + HTMLBugsTable = "bugs" + HTMLBugCountsTable = "bug_counts" + HTMLReprosTable = "repros" + HTMLCReprosTable = "crepros" + HTMLReproAttemptsTable = "repro_attempts" + HTMLReproDurationTable = "repro_duration" ) type uiTableGenerator = func(urlPrefix string, view StatView, r *http.Request) (*uiTable, error) @@ -170,6 +174,10 @@ func (ctx *TestbedContext) getTableTypes() []uiTableType { {HTMLStatsTable, "Statistics", ctx.httpMainStatsTable}, {HTMLBugsTable, "Bugs", ctx.genSimpleTableController((StatView).GenerateBugTable, true)}, {HTMLBugCountsTable, "Bug Counts", ctx.genSimpleTableController((StatView).GenerateBugCountsTable, false)}, + {HTMLReprosTable, "Repros", ctx.genSimpleTableController((StatView).GenerateReproSuccessTable, true)}, + {HTMLCReprosTable, "C Repros", ctx.genSimpleTableController((StatView).GenerateCReproSuccessTable, true)}, + {HTMLReproAttemptsTable, "All Repros", ctx.genSimpleTableController((StatView).GenerateReproAttemptsTable, false)}, + {HTMLReproDurationTable, "Duration", ctx.genSimpleTableController((StatView).GenerateReproDurationTable, true)}, } typeList := []uiTableType{} for _, t := range allTypeList { diff --git a/tools/syz-testbed/instance.go b/tools/syz-testbed/instance.go index f34df8a98..f3a4c0bc6 100644 --- a/tools/syz-testbed/instance.go +++ b/tools/syz-testbed/instance.go @@ -18,6 +18,7 @@ type Instance interface { Run() error Stop() FetchResult() (RunResult, error) + Uptime() time.Duration } // The essential information about an active instance. @@ -26,6 +27,8 @@ type InstanceCommon struct { LogFile string ExecCommand string ExecCommandArgs []string + StartedAt time.Time + StoppedAt time.Time stopChannel chan bool } @@ -45,14 +48,15 @@ func (inst *InstanceCommon) Run() error { } complete := make(chan error) + inst.StartedAt = time.Now() cmd.Start() go func() { complete <- cmd.Wait() }() select { - case <-complete: - return nil + case err := <-complete: + return err case <-inst.stopChannel: // TODO: handle other OSes? cmd.Process.Signal(os.Interrupt) @@ -65,8 +69,9 @@ func (inst *InstanceCommon) Run() error { cmd.Process.Kill() <-complete } - return nil } + inst.StoppedAt = time.Now() + return nil } func (inst *InstanceCommon) Stop() { @@ -76,6 +81,13 @@ func (inst *InstanceCommon) Stop() { } } +func (inst *InstanceCommon) Uptime() time.Duration { + if !inst.StartedAt.IsZero() && inst.StoppedAt.IsZero() { + return time.Since(inst.StartedAt) + } + return inst.StoppedAt.Sub(inst.StartedAt) +} + type SyzManagerInstance struct { InstanceCommon SyzkallerInfo @@ -175,3 +187,56 @@ func (t *SyzManagerTarget) newSyzManagerInstance(slotName, uniqName string, chec RunTime: t.config.RunTime.Duration, }, nil } + +type SyzReproInstance struct { + InstanceCommon + SyzkallerInfo + Input *SyzReproInput + ReproFile string + CReproFile string + // TODO: we also want to extract source and new bug titles +} + +func (inst *SyzReproInstance) FetchResult() (RunResult, error) { + return &SyzReproResult{ + Input: inst.Input, + ReproFound: osutil.IsExist(inst.ReproFile), + CReproFound: osutil.IsExist(inst.CReproFile), + Duration: inst.Uptime(), + }, nil +} + +func (t *SyzReproTarget) newSyzReproInstance(slotName, uniqName string, input *SyzReproInput, + checkout *Checkout) (Instance, error) { + folder := filepath.Join(checkout.Path, fmt.Sprintf("run-%s", uniqName)) + common, err := SetupSyzkallerInstance(slotName, folder, checkout) + if err != nil { + return nil, err + } + + reproFile := filepath.Join(folder, "repro.txt") + cReproFile := filepath.Join(folder, "crepro.txt") + newExecLog := filepath.Join(folder, "execution-log.txt") + err = osutil.CopyFile(input.Path, newExecLog) + if err != nil { + return nil, err + } + return &SyzReproInstance{ + InstanceCommon: InstanceCommon{ + Name: uniqName, + LogFile: filepath.Join(folder, "log.txt"), + ExecCommand: filepath.Join(checkout.Path, "bin", "syz-repro"), + ExecCommandArgs: []string{ + "-config", common.CfgFile, + "-output", reproFile, + "-crepro", cReproFile, + newExecLog, + }, + stopChannel: make(chan bool, 1), + }, + SyzkallerInfo: *common, + Input: input, + ReproFile: reproFile, + CReproFile: cReproFile, + }, nil +} diff --git a/tools/syz-testbed/stats.go b/tools/syz-testbed/stats.go index 77fed55b7..5ea3a2931 100644 --- a/tools/syz-testbed/stats.go +++ b/tools/syz-testbed/stats.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/google/syzkaller/pkg/osutil" "github.com/google/syzkaller/pkg/stats" @@ -27,6 +28,13 @@ type SyzManagerResult struct { StatRecords []StatRecord } +type SyzReproResult struct { + Input *SyzReproInput + ReproFound bool + CReproFound bool + Duration time.Duration +} + // A snapshot of syzkaller statistics at a particular time. type StatRecord map[string]uint64 @@ -183,6 +191,17 @@ func (group RunResultGroup) SyzManagerResults() []*SyzManagerResult { return ret } +func (group RunResultGroup) SyzReproResults() []*SyzReproResult { + ret := []*SyzReproResult{} + for _, rawRes := range group.Results { + res, ok := rawRes.(*SyzReproResult) + if ok { + ret = append(ret, res) + } + } + return ret +} + func (group RunResultGroup) AvgStatRecords() []map[string]uint64 { ret := []map[string]uint64{} commonLen := group.minResultLength() @@ -284,6 +303,99 @@ func (view StatView) InstanceStatsTable() (*Table, error) { return newView.StatsTable() } +// How often we find a repro to each crash log. +func (view StatView) GenerateReproSuccessTable() (*Table, error) { + table := NewTable("Bug") + for _, group := range view.Groups { + table.AddColumn(group.Name) + } + for _, group := range view.Groups { + for _, result := range group.SyzReproResults() { + title := result.Input.Title + cell, _ := table.Get(title, group.Name).(*RatioCell) + if cell == nil { + cell = NewRatioCell(0, 0) + } + cell.TotalCount++ + if result.ReproFound { + cell.TrueCount++ + } + table.Set(title, group.Name, cell) + } + } + return table, nil +} + +// What share of found repros also have a C repro. +func (view StatView) GenerateCReproSuccessTable() (*Table, error) { + table := NewTable("Bug") + for _, group := range view.Groups { + table.AddColumn(group.Name) + } + for _, group := range view.Groups { + for _, result := range group.SyzReproResults() { + if !result.ReproFound { + continue + } + title := result.Input.Title + cell, _ := table.Get(title, group.Name).(*RatioCell) + if cell == nil { + cell = NewRatioCell(0, 0) + } + cell.TotalCount++ + if result.CReproFound { + cell.TrueCount++ + } + table.Set(title, group.Name, cell) + } + } + return table, nil +} + +// What share of found repros also have a C repro. +func (view StatView) GenerateReproDurationTable() (*Table, error) { + table := NewTable("Bug") + for _, group := range view.Groups { + table.AddColumn(group.Name) + } + for _, group := range view.Groups { + samples := make(map[string]*stats.Sample) + for _, result := range group.SyzReproResults() { + title := result.Input.Title + var sample *stats.Sample + sample, ok := samples[title] + if !ok { + sample = &stats.Sample{} + samples[title] = sample + } + sample.Xs = append(sample.Xs, result.Duration.Seconds()) + } + + for title, sample := range samples { + table.Set(title, group.Name, NewValueCell(sample)) + } + } + return table, nil +} + +// List all repro attempts. +func (view StatView) GenerateReproAttemptsTable() (*Table, error) { + table := NewTable("Result #", "Bug", "Checkout", "Repro found", "C repro found", "Duration") + for gid, group := range view.Groups { + for rid, result := range group.SyzReproResults() { + table.AddRow( + fmt.Sprintf("%d-%d", gid, rid), + result.Input.Title, + group.Name, + NewBoolCell(result.ReproFound), + NewBoolCell(result.CReproFound), + result.Duration.Round(time.Second).String(), + ) + } + } + return table, nil +} + // Average bench files of several instances into a single bench file. func (group *RunResultGroup) SaveAvgBenchFile(fileName string) error { f, err := os.Create(fileName) @@ -318,7 +430,7 @@ func (view *StatView) SaveAvgBenches(benchDir string) ([]string, error) { func (view *StatView) IsEmpty() bool { for _, group := range view.Groups { - if group.minResultLength() > 0 { + if len(group.Results) > 0 { return false } } diff --git a/tools/syz-testbed/targets.go b/tools/syz-testbed/targets.go index c19dec2ca..97941d6ca 100644 --- a/tools/syz-testbed/targets.go +++ b/tools/syz-testbed/targets.go @@ -5,11 +5,14 @@ package main import ( "fmt" + "io/ioutil" "log" + "os" "path/filepath" "sync" "github.com/google/syzkaller/pkg/osutil" + "github.com/google/syzkaller/pkg/tool" ) // TestbedTarget represents all behavioral differences between specific testbed targets. @@ -32,6 +35,35 @@ var targetConstructors = map[string]func(cfg *TestbedConfig) TestbedTarget{ config: cfg, } }, + "syz-repro": func(cfg *TestbedConfig) TestbedTarget { + inputs := []*SyzReproInput{} + if cfg.InputLogs != "" { + err := filepath.Walk(cfg.InputLogs, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + inputs = append(inputs, &SyzReproInput{ + Path: path, + runBy: make(map[*Checkout]int), + }) + } + return nil + }) + if err != nil { + tool.Failf("failed to read logs file directory: %s", err) + } + // TODO: shuffle? + } + if len(inputs) == 0 { + tool.Failf("no inputs given") + } + return &SyzReproTarget{ + config: cfg, + dedupTitle: make(map[string]int), + inputs: inputs, + } + }, } func (t *SyzManagerTarget) NewJob(slotName string, checkouts []*Checkout) (*Checkout, Instance, error) { @@ -81,3 +113,115 @@ func (t *SyzManagerTarget) SaveStatView(view StatView, dir string) error { _, err = view.SaveAvgBenches(benchDir) return err } + +// TODO: consider other repro testing modes. +// E.g. group different logs by title. Then we could also set different sets of inputs +// for each checkout. It can be important if we improve executor logging. +type SyzReproTarget struct { + config *TestbedConfig + inputs []*SyzReproInput + seqID int + dedupTitle map[string]int + mu sync.Mutex +} + +type SyzReproInput struct { + Path string + Title string + Skip bool + origTitle string + runBy map[*Checkout]int +} + +func (inp *SyzReproInput) QueryTitle(checkout *Checkout, dupsMap map[string]int) error { + data, err := ioutil.ReadFile(inp.Path) + if err != nil { + return fmt.Errorf("failed to read: %s", err) + } + report := checkout.GetReporter().Parse(data) + if report == nil { + return fmt.Errorf("found no crash") + } + if inp.Title == "" { + inp.origTitle = report.Title + inp.Title = report.Title + // Some bug titles may be present in multiple log files. + // Ensure they are all distict to the user. + dupsMap[inp.origTitle]++ + if dupsMap[inp.Title] > 1 { + inp.Title += fmt.Sprintf(" (%d)", dupsMap[inp.origTitle]) + } + } + return nil +} + +func (t *SyzReproTarget) NewJob(slotName string, checkouts []*Checkout) (*Checkout, Instance, error) { + t.mu.Lock() + seqID := t.seqID + checkout := checkouts[t.seqID%len(checkouts)] + t.seqID++ + // This may be not the most efficient algorithm, but it guarantees even distribution of + // resources and CPU time is negligible in comparison with the amount of time each instance runs. + var input *SyzReproInput + for _, candidate := range t.inputs { + if candidate.Skip { + continue + } + if candidate.runBy[checkout] == 0 { + // This is the first time we'll attempt to give this log to the checkout. + // Check if it can handle it. + err := candidate.QueryTitle(checkout, t.dedupTitle) + if err != nil { + log.Printf("[log %s]: %s, skipping", candidate.Path, err) + candidate.Skip = true + continue + } + } + if input == nil || input.runBy[checkout] > candidate.runBy[checkout] { + // Pick the least executed one. + input = candidate + } + } + + if input == nil { + t.mu.Unlock() + return nil, nil, fmt.Errorf("no available inputs") + } + input.runBy[checkout]++ + t.mu.Unlock() + + uniqName := fmt.Sprintf("%s-%d", checkout.Name, seqID) + instance, err := t.newSyzReproInstance(slotName, uniqName, input, checkout) + if err != nil { + return nil, nil, err + } + return checkout, instance, nil +} + +func (t *SyzReproTarget) SupportsHTMLView(key string) bool { + supported := map[string]bool{ + HTMLReprosTable: true, + HTMLCReprosTable: true, + HTMLReproAttemptsTable: true, + HTMLReproDurationTable: true, + } + return supported[key] +} + +func (t *SyzReproTarget) SaveStatView(view StatView, dir string) error { + tableStats := map[string]func(view StatView) (*Table, error){ + "repro_success.csv": (StatView).GenerateReproSuccessTable, + "crepros_success.csv": (StatView).GenerateCReproSuccessTable, + "repro_attempts.csv": (StatView).GenerateReproAttemptsTable, + "repro_duration.csv": (StatView).GenerateReproDurationTable, + } + for fileName, genFunc := range tableStats { + table, err := genFunc(view) + if err == nil { + table.SaveAsCsv(filepath.Join(dir, fileName)) + } else { + log.Printf("stat generation error: %s", err) + } + } + return nil +} diff --git a/tools/syz-testbed/testbed.go b/tools/syz-testbed/testbed.go index 6f7a1868d..0af48bc1d 100644 --- a/tools/syz-testbed/testbed.go +++ b/tools/syz-testbed/testbed.go @@ -33,6 +33,7 @@ type TestbedConfig struct { Target string `json:"target"` // what application to test MaxInstances int `json:"max_instances"` // max # of simultaneously running instances RunTime DurationConfig `json:"run_time"` // lifetime of an instance (default "24h") + InputLogs string `json:"input_logs"` // folder with logs to be processed by syz-repro HTTP string `json:"http"` // on which port to set up a simple web dashboard BenchCmp string `json:"benchcmp"` // path to the syz-benchcmp executable Corpus string `json:"corpus"` // path to the corpus file -- cgit mrf-deployment