diff options
| -rw-r--r-- | pkg/instance/execprog.go | 26 | ||||
| -rw-r--r-- | pkg/mgrconfig/config.go | 5 | ||||
| -rw-r--r-- | pkg/mgrconfig/load.go | 6 | ||||
| -rw-r--r-- | pkg/repro/strace.go | 67 | ||||
| -rw-r--r-- | syz-manager/html.go | 8 | ||||
| -rw-r--r-- | syz-manager/manager.go | 100 | ||||
| -rw-r--r-- | tools/syz-crush/crush.go | 7 | ||||
| -rw-r--r-- | tools/syz-repro/repro.go | 18 |
8 files changed, 208 insertions, 29 deletions
diff --git a/pkg/instance/execprog.go b/pkg/instance/execprog.go index 1d775c22c..f6f9bfa01 100644 --- a/pkg/instance/execprog.go +++ b/pkg/instance/execprog.go @@ -13,6 +13,7 @@ import ( "github.com/google/syzkaller/pkg/osutil" "github.com/google/syzkaller/pkg/report" "github.com/google/syzkaller/prog" + "github.com/google/syzkaller/sys/targets" "github.com/google/syzkaller/vm" ) @@ -23,6 +24,7 @@ type OptionalConfig struct { Logf ExecutorLogger OldFlagsCompatMode bool BeforeContextLen int + StraceBin string } type ExecProgInstance struct { @@ -62,6 +64,14 @@ func SetupExecProg(vmInst *vm.Instance, mgrCfg *mgrconfig.Config, reporter *repo } if opt != nil { ret.OptionalConfig = *opt + if ret.StraceBin != "" { + var err error + ret.StraceBin, err = vmInst.Copy(ret.StraceBin) + if err != nil { + vmInst.Close() + return nil, &TestError{Title: fmt.Sprintf("failed to copy strace bin: %v", err)} + } + } } if ret.Logf == nil { ret.Logf = func(int, string, ...interface{}) {} @@ -87,6 +97,19 @@ func CreateExecProgInstance(vmPool *vm.Pool, vmIndex int, mgrCfg *mgrconfig.Conf } func (inst *ExecProgInstance) runCommand(command string, duration time.Duration) (*RunResult, error) { + var prefixOutput []byte + if inst.StraceBin != "" { + filterCalls := "" + switch inst.mgrCfg.SysTarget.OS { + case targets.Linux: + // wait4 and nanosleep generate a lot of noise, especially when running syz-executor. + // We cut them on the VM side in order to decrease load on the network and to use + // the limited buffer size wisely. + filterCalls = ` -e \!wait4,clock_nanosleep,nanosleep` + } + command = inst.StraceBin + filterCalls + ` -s 100 -x -f ` + command + prefixOutput = []byte(fmt.Sprintf("%s\n\n<...>\n", command)) + } outc, errc, err := inst.VMInstance.Run(duration, nil, command) if err != nil { return nil, fmt.Errorf("failed to run command in VM: %v", err) @@ -95,6 +118,9 @@ func (inst *ExecProgInstance) runCommand(command string, duration time.Duration) ExecutionResult: *inst.VMInstance.MonitorExecutionRaw(outc, errc, inst.reporter, inst.ExitCondition, inst.BeforeContextLen), } + if len(prefixOutput) > 0 { + result.RawOutput = append(prefixOutput, result.RawOutput...) + } if result.Report == nil { inst.Logf(2, "program did not crash") } else { diff --git a/pkg/mgrconfig/config.go b/pkg/mgrconfig/config.go index c13e6f452..20a0b7b57 100644 --- a/pkg/mgrconfig/config.go +++ b/pkg/mgrconfig/config.go @@ -168,6 +168,11 @@ type Config struct { // Regexps are matched against bug title, guilty file and maintainer emails. Interests []string `json:"interests,omitempty"` + // Path to the strace binary compiled for the target architecture. + // If set, for each reproducer syzkaller will run it once more under strace and save + // the output. + StraceBin string `json:"strace_bin"` + // Type of virtual machine to use, e.g. "qemu", "gce", "android", "isolated", etc. Type string `json:"type"` // VM-type-specific parameters. diff --git a/pkg/mgrconfig/load.go b/pkg/mgrconfig/load.go index c9528b38a..2b49d0cbd 100644 --- a/pkg/mgrconfig/load.go +++ b/pkg/mgrconfig/load.go @@ -256,6 +256,12 @@ func (cfg *Config) completeBinaries() error { if cfg.ExecutorBin != "" && !osutil.IsExist(cfg.ExecutorBin) { return fmt.Errorf("bad config syzkaller param: can't find %v", cfg.ExecutorBin) } + if cfg.StraceBin != "" { + if !osutil.IsExist(cfg.StraceBin) { + return fmt.Errorf("bad config param strace_bin: can't find %v", cfg.StraceBin) + } + cfg.StraceBin = osutil.Abs(cfg.StraceBin) + } return nil } diff --git a/pkg/repro/strace.go b/pkg/repro/strace.go new file mode 100644 index 000000000..07c3ea025 --- /dev/null +++ b/pkg/repro/strace.go @@ -0,0 +1,67 @@ +// Copyright 2022 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package repro + +import ( + "fmt" + + "github.com/google/syzkaller/pkg/instance" + "github.com/google/syzkaller/pkg/log" + "github.com/google/syzkaller/pkg/mgrconfig" + "github.com/google/syzkaller/pkg/report" + "github.com/google/syzkaller/vm" +) + +type StraceResult struct { + Report *report.Report + Output []byte + Error error +} + +const ( + straceOutputLogSize = 2048 << 10 +) + +func RunStrace(result *Result, cfg *mgrconfig.Config, reporter *report.Reporter, + vmPool *vm.Pool, vmIndex int) *StraceResult { + if cfg.StraceBin == "" { + return straceFailed(fmt.Errorf("strace binary is not set in the config")) + } + inst, err := instance.CreateExecProgInstance(vmPool, vmIndex, cfg, reporter, + &instance.OptionalConfig{ + StraceBin: cfg.StraceBin, + BeforeContextLen: straceOutputLogSize, + }) + if err != nil { + return straceFailed(fmt.Errorf("failed to set up instance: %v", err)) + } + defer inst.VMInstance.Close() + + var runRes *instance.RunResult + if result.CRepro { + log.Logf(1, "running C repro under strace") + runRes, err = inst.RunCProg(result.Prog, result.Duration, result.Opts) + } else { + log.Logf(1, "running syz repro under strace") + runRes, err = inst.RunSyzProg(result.Prog.Serialize(), result.Duration, result.Opts) + } + if err != nil { + return straceFailed(fmt.Errorf("failed to generate strace log: %v", err)) + } + return &StraceResult{ + Report: runRes.Report, + Output: runRes.RawOutput, + } +} + +func straceFailed(err error) *StraceResult { + return &StraceResult{Error: err} +} + +func (strace *StraceResult) IsSameBug(repro *Result) bool { + if strace == nil || strace.Report == nil || repro.Report == nil { + return false + } + return strace.Report.Title == repro.Report.Title +} diff --git a/syz-manager/html.go b/syz-manager/html.go index 781ba3279..6252f55fa 100644 --- a/syz-manager/html.go +++ b/syz-manager/html.go @@ -609,6 +609,7 @@ func readCrash(workdir, dir string, repros map[string]bool, start time.Time, ful var crashes []*UICrash reproAttempts := 0 hasRepro, hasCRepro := false, false + strace := "" reports := make(map[string]bool) for _, f := range files { if strings.HasPrefix(f, "log") { @@ -627,6 +628,8 @@ func readCrash(workdir, dir string, repros map[string]bool, start time.Time, ful } else if f == "repro.report" { } else if f == "repro0" || f == "repro1" || f == "repro2" { reproAttempts++ + } else if f == "strace.log" { + strace = filepath.Join("crashes", dir, f) } } @@ -658,6 +661,7 @@ func readCrash(workdir, dir string, repros map[string]bool, start time.Time, ful ID: dir, Count: len(crashes), Triaged: triaged, + Strace: strace, Crashes: crashes, } } @@ -713,6 +717,7 @@ type UICrashType struct { ID string Count int Triaged string + Strace string Crashes []*UICrash } @@ -794,6 +799,9 @@ var summaryTemplate = html.CreatePage(` {{if $c.Triaged}} <a href="/report?id={{$c.ID}}">{{$c.Triaged}}</a> {{end}} + {{if $c.Strace}} + <a href="/file?name={{$c.Strace}}">Strace</a> + {{end}} </td> </tr> {{end}} diff --git a/syz-manager/manager.go b/syz-manager/manager.go index e34e59322..adc4b2556 100644 --- a/syz-manager/manager.go +++ b/syz-manager/manager.go @@ -296,7 +296,8 @@ type RunResult struct { type ReproResult struct { instances []int report0 *report.Report // the original report we started reproducing - res *repro.Result + repro *repro.Result + strace *repro.StraceResult stats *repro.Stats err error hub bool // repro came from hub @@ -361,7 +362,7 @@ func (mgr *Manager) vmLoop() { atomic.AddUint32(&mgr.numReproducing, 1) log.Logf(1, "loop: starting repro of '%v' on instances %+v", crash.Title, vmIndexes) go func() { - reproDone <- mgr.runRepro(crash, vmIndexes) + reproDone <- mgr.runRepro(crash, vmIndexes, instances.Put) }() } for !canRepro() { @@ -409,23 +410,22 @@ func (mgr *Manager) vmLoop() { atomic.AddUint32(&mgr.numReproducing, ^uint32(0)) crepro := false title := "" - if res.res != nil { - crepro = res.res.CRepro - title = res.res.Report.Title + if res.repro != nil { + crepro = res.repro.CRepro + title = res.repro.Report.Title } log.Logf(1, "loop: repro on %+v finished '%v', repro=%v crepro=%v desc='%v'", - res.instances, res.report0.Title, res.res != nil, crepro, title) + res.instances, res.report0.Title, res.repro != nil, crepro, title) if res.err != nil { log.Logf(0, "repro failed: %v", res.err) } delete(reproducing, res.report0.Title) - instances.Put(res.instances...) - if res.res == nil { + if res.repro == nil { if !res.hub { mgr.saveFailedRepro(res.report0, res.stats) } } else { - mgr.saveRepro(res.res, res.stats, res.hub) + mgr.saveRepro(res) } case <-shutdown: log.Logf(1, "loop: shutting down...") @@ -448,17 +448,39 @@ func (mgr *Manager) vmLoop() { } } -func (mgr *Manager) runRepro(crash *Crash, vmIndexes []int) *ReproResult { +func (mgr *Manager) runRepro(crash *Crash, vmIndexes []int, putInstances func(...int)) *ReproResult { features := mgr.checkResult.Features res, stats, err := repro.Run(crash.Output, mgr.cfg, features, mgr.reporter, mgr.vmPool, vmIndexes) - return &ReproResult{ + ret := &ReproResult{ instances: vmIndexes, report0: crash.Report, - res: res, + repro: res, stats: stats, err: err, hub: crash.hub, } + if err != nil && res != nil && mgr.cfg.StraceBin != "" { + // We need only one instance to get strace output, release the rest. + putInstances(vmIndexes[1:]...) + defer putInstances(vmIndexes[0]) + + const straceAttempts = 2 + for i := 1; i <= straceAttempts; i++ { + strace := repro.RunStrace(res, mgr.cfg, mgr.reporter, mgr.vmPool, vmIndexes[0]) + sameBug := strace.IsSameBug(res) + log.Logf(0, "strace run attempt %d/%d for '%s': same bug %v, error %v", + i, straceAttempts, res.Report.Title, sameBug, strace.Error) + // We only want to save strace output if it resulted in the same bug. + // Otherwise, it will be hard to reproduce on syzbot and will confuse users. + if sameBug { + ret.strace = strace + break + } + } + } else { + putInstances(vmIndexes...) + } + return ret } type ResourcePool struct { @@ -973,23 +995,23 @@ func (mgr *Manager) saveFailedRepro(rep *report.Report, stats *repro.Stats) { } } -func (mgr *Manager) saveRepro(res *repro.Result, stats *repro.Stats, hub bool) { - rep := res.Report - opts := fmt.Sprintf("# %+v\n", res.Opts) - prog := res.Prog.Serialize() +func (mgr *Manager) saveRepro(res *ReproResult) { + repro := res.repro + opts := fmt.Sprintf("# %+v\n", repro.Opts) + prog := repro.Prog.Serialize() // Append this repro to repro list to send to hub if it didn't come from hub originally. - if !hub { + if !res.hub { progForHub := []byte(fmt.Sprintf("# %+v\n# %v\n# %v\n%s", - res.Opts, res.Report.Title, mgr.cfg.Tag, prog)) + repro.Opts, repro.Report.Title, mgr.cfg.Tag, prog)) mgr.mu.Lock() mgr.newRepros = append(mgr.newRepros, progForHub) mgr.mu.Unlock() } var cprogText []byte - if res.CRepro { - cprog, err := csource.Write(res.Prog, res.Opts) + if repro.CRepro { + cprog, err := csource.Write(repro.Prog, repro.Opts) if err == nil { formatted, err := csource.Format(cprog) if err == nil { @@ -1007,16 +1029,25 @@ func (mgr *Manager) saveRepro(res *repro.Result, stats *repro.Stats, hub bool) { // 2. Repro re-tried 3 times and still got corrupted report at the end, // so maybe corrupted report detection is broken. // 3. Reproduction is expensive so it's good to persist the result. + + report := repro.Report + output := report.Output + if res.strace != nil { + // If syzkaller managed to successfully run the repro with strace, send + // the report and the output generated under strace. + report = res.strace.Report + output = res.strace.Output + } dc := &dashapi.Crash{ BuildID: mgr.cfg.Tag, - Title: res.Report.Title, - AltTitles: res.Report.AltTitles, - Suppressed: res.Report.Suppressed, - Recipients: res.Report.Recipients.ToDash(), - Log: res.Report.Output, - Report: res.Report.Report, - ReproOpts: res.Opts.Serialize(), - ReproSyz: res.Prog.Serialize(), + Title: report.Title, + AltTitles: report.AltTitles, + Suppressed: report.Suppressed, + Recipients: report.Recipients.ToDash(), + Log: output, + Report: report.Report, + ReproOpts: repro.Opts.Serialize(), + ReproSyz: repro.Prog.Serialize(), ReproC: cprogText, } if _, err := mgr.dash.ReportCrash(dc); err != nil { @@ -1028,6 +1059,7 @@ func (mgr *Manager) saveRepro(res *repro.Result, stats *repro.Stats, hub bool) { } } + rep := repro.Report dir := filepath.Join(mgr.crashdir, hash.String([]byte(rep.Title))) osutil.MkdirAll(dir) @@ -1047,7 +1079,17 @@ func (mgr *Manager) saveRepro(res *repro.Result, stats *repro.Stats, hub bool) { if len(cprogText) > 0 { osutil.WriteFile(filepath.Join(dir, "repro.cprog"), cprogText) } - saveReproStats(filepath.Join(dir, "repro.stats"), stats) + if res.strace != nil { + // Unlike dashboard reporting, we save strace output separately from the original log. + if res.strace.Error != nil { + osutil.WriteFile(filepath.Join(dir, "strace.error"), + []byte(fmt.Sprintf("%v", res.strace.Error))) + } + if len(res.strace.Output) > 0 { + osutil.WriteFile(filepath.Join(dir, "strace.log"), res.strace.Output) + } + } + saveReproStats(filepath.Join(dir, "repro.stats"), res.stats) } func saveReproStats(filename string, stats *repro.Stats) { diff --git a/tools/syz-crush/crush.go b/tools/syz-crush/crush.go index 36424c61a..a88041a37 100644 --- a/tools/syz-crush/crush.go +++ b/tools/syz-crush/crush.go @@ -31,6 +31,7 @@ var ( flagDebug = flag.Bool("debug", false, "dump all VM output to console") flagRestartTime = flag.Duration("restart_time", 0, "how long to run the test") flagInfinite = flag.Bool("infinite", true, "by default test is run for ever, -infinite=false to stop on crash") + flagStrace = flag.Bool("strace", false, "run under strace (binary must be set in the config file") ) type FileType int @@ -59,6 +60,9 @@ func main() { } else { log.Printf("running until crash is found or till %v", *flagRestartTime) } + if *flagStrace && cfg.StraceBin == "" { + log.Fatalf("strace_bin must not be empty in order to run with -strace") + } vmPool, err := vm.Create(cfg, *flagDebug) if err != nil { @@ -160,6 +164,9 @@ func runInstance(cfg *mgrconfig.Config, reporter *report.Reporter, optArgs := &instance.OptionalConfig{ ExitCondition: vm.ExitTimeout, } + if *flagStrace { + optArgs.StraceBin = cfg.StraceBin + } var err error inst, err := instance.CreateExecProgInstance(vmPool, index, cfg, reporter, optArgs) if err != nil { diff --git a/tools/syz-repro/repro.go b/tools/syz-repro/repro.go index 6622b8203..247cc11cd 100644 --- a/tools/syz-repro/repro.go +++ b/tools/syz-repro/repro.go @@ -25,6 +25,7 @@ var ( flagDebug = flag.Bool("debug", false, "print debug output") flagOutput = flag.String("output", filepath.Join(".", "repro.txt"), "output description file (output.txt)") flagCRepro = flag.String("crepro", filepath.Join(".", "repro.c"), "output c file (repro.c)") + flagStrace = flag.String("strace", "", "output strace log (strace_bin must be set)") ) func main() { @@ -104,4 +105,21 @@ func main() { log.Logf(0, "failed to write C repro to file: %v", err) } } + if *flagStrace != "" { + result := repro.RunStrace(res, cfg, reporter, vmPool, vmIndexes[0]) + if result.Error != nil { + log.Logf(0, "failed to run strace: %v", result.Error) + } else { + if result.Report != nil { + log.Logf(0, "under strace repro crashed with title: %s", result.Report.Title) + } else { + log.Logf(0, "repro didn't crash under strace") + } + if err := osutil.WriteFile(*flagStrace, result.Output); err == nil { + fmt.Printf("C file saved to %s\n", *flagStrace) + } else { + log.Logf(0, "failed to write strace output to file: %v", err) + } + } + } } |
