From 5153aeaffd096514c1f2652c69cd0fc0d298b1d3 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Wed, 29 Nov 2017 13:23:42 +0100 Subject: syz-ci: test images before using them Boot and minimally test images before declaring them as good and switching to using them. If image build/boot/test fails, upload report about this to dashboard. --- dashboard/app/api.go | 78 ++++++++---- dashboard/app/entities.go | 10 ++ dashboard/app/jobs.go | 2 +- dashboard/app/jobs_test.go | 2 +- dashboard/app/mail_test_result.txt | 2 +- dashboard/app/reporting_external.go | 4 +- dashboard/dashapi/dashapi.go | 9 ++ pkg/ipc/ipc.go | 5 +- pkg/ipc/ipc_test.go | 3 + prog/rand.go | 9 ++ syz-ci/jobs.go | 29 +++-- syz-ci/manager.go | 230 +++++++++++++++++++++++++++--------- syz-ci/syz-ci.go | 1 + syz-ci/testing.go | 103 ++++++++++++++++ syz-fuzzer/fuzzer.go | 12 +- syz-fuzzer/testing.go | 72 +++++++++++ vm/gce/gce.go | 2 +- vm/qemu/qemu.go | 4 +- vm/vm.go | 9 +- vm/vmimpl/vmimpl.go | 10 ++ 20 files changed, 489 insertions(+), 107 deletions(-) create mode 100644 syz-ci/testing.go create mode 100644 syz-fuzzer/testing.go diff --git a/dashboard/app/api.go b/dashboard/app/api.go index a3a466ce6..db265f8b8 100644 --- a/dashboard/app/api.go +++ b/dashboard/app/api.go @@ -29,20 +29,25 @@ func init() { } var apiHandlers = map[string]APIHandler{ - "log_error": apiLogError, + "log_error": apiLogError, + "job_poll": apiJobPoll, + "job_done": apiJobDone, + "reporting_poll": apiReportingPoll, + "reporting_update": apiReportingUpdate, +} + +var apiNamespaceHandlers = map[string]APINamespaceHandler{ "upload_build": apiUploadBuild, "builder_poll": apiBuilderPoll, - "job_poll": apiJobPoll, - "job_done": apiJobDone, + "report_build_error": apiReportBuildError, "report_crash": apiReportCrash, "report_failed_repro": apiReportFailedRepro, "need_repro": apiNeedRepro, - "reporting_poll": apiReportingPoll, - "reporting_update": apiReportingUpdate, } type JSONHandler func(c context.Context, r *http.Request) (interface{}, error) -type APIHandler func(c context.Context, ns string, r *http.Request) (interface{}, error) +type APIHandler func(c context.Context, r *http.Request) (interface{}, error) +type APINamespaceHandler func(c context.Context, ns string, r *http.Request) (interface{}, error) // Overridable for testing. var timeNow = func(c context.Context) time.Time { @@ -82,10 +87,17 @@ func handleAPI(c context.Context, r *http.Request) (reply interface{}, err error } method := r.FormValue("method") handler := apiHandlers[method] - if handler == nil { + if handler != nil { + return handler(c, r) + } + nsHandler := apiNamespaceHandlers[method] + if nsHandler == nil { return nil, fmt.Errorf("unknown api method %q", method) } - return handler(c, ns, r) + if ns == "" { + return nil, fmt.Errorf("method %q must be called within a namespace", method) + } + return nsHandler(c, ns, r) } func checkClient(c context.Context, name0, key0 string) (string, error) { @@ -110,7 +122,7 @@ func checkClient(c context.Context, name0, key0 string) (string, error) { return "", fmt.Errorf("unauthorized api request from %q", name0) } -func apiLogError(c context.Context, ns string, r *http.Request) (interface{}, error) { +func apiLogError(c context.Context, r *http.Request) (interface{}, error) { req := new(dashapi.LogEntry) if err := json.NewDecoder(r.Body).Decode(req); err != nil { return nil, fmt.Errorf("failed to unmarshal request: %v", err) @@ -159,7 +171,7 @@ loop: return resp, nil } -func apiJobPoll(c context.Context, ns string, r *http.Request) (interface{}, error) { +func apiJobPoll(c context.Context, r *http.Request) (interface{}, error) { req := new(dashapi.JobPollReq) if err := json.NewDecoder(r.Body).Decode(req); err != nil { return nil, fmt.Errorf("failed to unmarshal request: %v", err) @@ -170,7 +182,7 @@ func apiJobPoll(c context.Context, ns string, r *http.Request) (interface{}, err return pollPendingJobs(c, req.Managers) } -func apiJobDone(c context.Context, ns string, r *http.Request) (interface{}, error) { +func apiJobDone(c context.Context, r *http.Request) (interface{}, error) { req := new(dashapi.JobDoneReq) if err := json.NewDecoder(r.Body).Decode(req); err != nil { return nil, fmt.Errorf("failed to unmarshal request: %v", err) @@ -184,11 +196,22 @@ func apiUploadBuild(c context.Context, ns string, r *http.Request) (interface{}, if err := json.NewDecoder(r.Body).Decode(req); err != nil { return nil, fmt.Errorf("failed to unmarshal request: %v", err) } - err := uploadBuild(c, ns, req) - return nil, err + if err := uploadBuild(c, ns, req, BuildNormal); err != nil { + return nil, err + } + if len(req.Commits) != 0 { + if err := addCommitsToBugs(c, ns, req.Manager, req.Commits); err != nil { + return nil, err + } + } + return nil, nil } -func uploadBuild(c context.Context, ns string, req *dashapi.Build) error { +func uploadBuild(c context.Context, ns string, req *dashapi.Build, typ BuildType) error { + if _, err := loadBuild(c, ns, req.ID); err == nil { + return nil + } + checkStrLen := func(str, name string, maxLen int) error { if str == "" { return fmt.Errorf("%v is empty", name) @@ -227,6 +250,8 @@ func uploadBuild(c context.Context, ns string, req *dashapi.Build) error { Namespace: ns, Manager: req.Manager, ID: req.ID, + Type: typ, + Time: timeNow(c), OS: req.OS, Arch: req.Arch, VMArch: req.VMArch, @@ -240,13 +265,6 @@ func uploadBuild(c context.Context, ns string, req *dashapi.Build) error { if _, err := datastore.Put(c, buildKey(c, ns, req.ID), build); err != nil { return err } - - if len(req.Commits) != 0 { - if err := addCommitsToBugs(c, ns, req.Manager, req.Commits); err != nil { - return err - } - } - return nil } @@ -348,6 +366,20 @@ func stringInList(list []string, str string) bool { return false } +func apiReportBuildError(c context.Context, ns string, r *http.Request) (interface{}, error) { + req := new(dashapi.BuildErrorReq) + if err := json.NewDecoder(r.Body).Decode(req); err != nil { + return nil, fmt.Errorf("failed to unmarshal request: %v", err) + } + if err := uploadBuild(c, ns, &req.Build, BuildFailed); err != nil { + return nil, err + } + if _, err := reportCrash(c, ns, &req.Crash); err != nil { + return nil, err + } + return nil, nil +} + const corruptedReportTitle = "corrupted report" func apiReportCrash(c context.Context, ns string, r *http.Request) (interface{}, error) { @@ -355,6 +387,10 @@ func apiReportCrash(c context.Context, ns string, r *http.Request) (interface{}, if err := json.NewDecoder(r.Body).Decode(req); err != nil { return nil, fmt.Errorf("failed to unmarshal request: %v", err) } + return reportCrash(c, ns, req) +} + +func reportCrash(c context.Context, ns string, req *dashapi.Crash) (interface{}, error) { req.Title = limitLength(req.Title, maxTextLen) req.Maintainers = email.MergeEmailLists(req.Maintainers) if req.Corrupted { diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go index 6c63ef5e3..6661b84b7 100644 --- a/dashboard/app/entities.go +++ b/dashboard/app/entities.go @@ -28,6 +28,8 @@ type Build struct { Namespace string Manager string ID string // unique ID generated by syz-ci + Type BuildType + Time time.Time OS string Arch string VMArch string @@ -153,6 +155,14 @@ const ( ReproLevelC = dashapi.ReproLevelC ) +type BuildType int + +const ( + BuildNormal BuildType = iota + BuildFailed + BuildJob +) + func buildKey(c context.Context, ns, id string) *datastore.Key { if ns == "" { panic("requesting build key outside of namespace") diff --git a/dashboard/app/jobs.go b/dashboard/app/jobs.go index 94f1400d8..42743c78d 100644 --- a/dashboard/app/jobs.go +++ b/dashboard/app/jobs.go @@ -244,7 +244,7 @@ func doneJob(c context.Context, req *dashapi.JobDoneReq) error { return fmt.Errorf("job %v: already finished", jobID) } ns := job.Namespace - if err := uploadBuild(c, ns, &req.Build); err != nil { + if err := uploadBuild(c, ns, &req.Build, BuildJob); err != nil { return err } if job.Error, err = putText(c, ns, "Error", req.Error, false); err != nil { diff --git a/dashboard/app/jobs_test.go b/dashboard/app/jobs_test.go index af997f69b..f8d82ee22 100644 --- a/dashboard/app/jobs_test.go +++ b/dashboard/app/jobs_test.go @@ -145,7 +145,7 @@ Raw console output is attached. c.expectEQ(msg.Attachments[1].Data, []byte(patch)) c.expectEQ(msg.Body, `Hello, -syzbot tried to test the proposed patch but build failed: +syzbot tried to test the proposed patch but build/boot failed: failed to apply patch diff --git a/dashboard/app/mail_test_result.txt b/dashboard/app/mail_test_result.txt index 83009298a..9219d1298 100644 --- a/dashboard/app/mail_test_result.txt +++ b/dashboard/app/mail_test_result.txt @@ -5,7 +5,7 @@ syzbot has tested the proposed patch but the reproducer still triggered crash: {{printf "%s" .Report}} {{else if .Error}} -syzbot tried to test the proposed patch but build failed: +syzbot tried to test the proposed patch but build/boot failed: {{printf "%s" .Error}} {{if .ErrorTruncated}} diff --git a/dashboard/app/reporting_external.go b/dashboard/app/reporting_external.go index 56f8c3b4c..c5e7c06fa 100644 --- a/dashboard/app/reporting_external.go +++ b/dashboard/app/reporting_external.go @@ -24,7 +24,7 @@ func (cfg *ExternalConfig) Type() string { return cfg.ID } -func apiReportingPoll(c context.Context, ns string, r *http.Request) (interface{}, error) { +func apiReportingPoll(c context.Context, r *http.Request) (interface{}, error) { req := new(dashapi.PollRequest) if err := json.NewDecoder(r.Body).Decode(req); err != nil { return nil, fmt.Errorf("failed to unmarshal request: %v", err) @@ -36,7 +36,7 @@ func apiReportingPoll(c context.Context, ns string, r *http.Request) (interface{ return resp, nil } -func apiReportingUpdate(c context.Context, ns string, r *http.Request) (interface{}, error) { +func apiReportingUpdate(c context.Context, r *http.Request) (interface{}, error) { req := new(dashapi.BugUpdate) if err := json.NewDecoder(r.Body).Decode(req); err != nil { return nil, fmt.Errorf("failed to unmarshal request: %v", err) diff --git a/dashboard/dashapi/dashapi.go b/dashboard/dashapi/dashapi.go index 50c144dae..8ebea29ac 100644 --- a/dashboard/dashapi/dashapi.go +++ b/dashboard/dashapi/dashapi.go @@ -121,6 +121,15 @@ func (dash *Dashboard) JobDone(req *JobDoneReq) error { return dash.query("job_done", req, nil) } +type BuildErrorReq struct { + Build Build + Crash Crash +} + +func (dash *Dashboard) ReportBuildError(req *BuildErrorReq) error { + return dash.query("report_build_error", req, nil) +} + // Crash describes a single kernel crash (potentially with repro). type Crash struct { BuildID string // refers to Build.ID diff --git a/pkg/ipc/ipc.go b/pkg/ipc/ipc.go index 2f1cf3ef1..f6576342f 100644 --- a/pkg/ipc/ipc.go +++ b/pkg/ipc/ipc.go @@ -308,8 +308,7 @@ func (env *Env) Exec(opts *ExecOpts, p *prog.Prog) (output []byte, info []CallIn if env.config.Flags&FlagUseShmem == 0 { progData = env.in[:progSize] } - needOutput := env.config.Flags&FlagSignal != 0 || opts.Flags&FlagCollectComps != 0 - if needOutput && env.out != nil { + if env.out != nil { // Zero out the first two words (ncmd and nsig), so that we don't have garbage there // if executor crashes before writing non-garbage there. for i := 0; i < 4; i++ { @@ -333,7 +332,7 @@ func (env *Env) Exec(opts *ExecOpts, p *prog.Prog) (output []byte, info []CallIn return } - if needOutput && env.out != nil { + if env.out != nil { info, err0 = env.readOutCoverage(p) } return diff --git a/pkg/ipc/ipc_test.go b/pkg/ipc/ipc_test.go index 51e961119..6d1c2c80e 100644 --- a/pkg/ipc/ipc_test.go +++ b/pkg/ipc/ipc_test.go @@ -114,6 +114,9 @@ func TestExecute(t *testing.T) { for i := 0; i < iters/len(flags); i++ { p := target.Generate(rs, 10, nil) + if i == 0 { + p = target.GenerateSimpleProg() + } opts := &ExecOpts{} output, _, _, _, err := env.Exec(opts, p) if err != nil { diff --git a/prog/rand.go b/prog/rand.go index 2b7709907..839c6d1c3 100644 --- a/prog/rand.go +++ b/prog/rand.go @@ -499,6 +499,15 @@ func (target *Target) GenerateAllSyzProg(rs rand.Source) *Prog { return p } +// GenerateSimpleProg generates the simplest non-empty program for testing +// (e.g. containing a single mmap). +func (target *Target) GenerateSimpleProg() *Prog { + return &Prog{ + Target: target, + Calls: []*Call{target.MakeMmap(0, 1)}, + } +} + func (r *randGen) generateArgs(s *state, types []Type) ([]Arg, []*Call) { var calls []*Call args := make([]Arg, len(types)) diff --git a/syz-ci/jobs.go b/syz-ci/jobs.go index 1a7eb14af..604dd1895 100644 --- a/syz-ci/jobs.go +++ b/syz-ci/jobs.go @@ -220,7 +220,6 @@ func (job *Job) buildImage() error { if err != nil { return fmt.Errorf("image build failed: %v", err) } - // TODO(dvyukov): test that the image is good (boots and we can ssh into it). mgrcfg := new(mgrconfig.Config) *mgrcfg = *mgr.managercfg @@ -247,17 +246,28 @@ func (job *Job) test() error { req, mgrcfg := job.req, job.mgrcfg Logf(0, "job: booting VM...") - vmEnv := mgrconfig.CreateVMEnv(mgrcfg, false) - vmPool, err := vm.Create(mgrcfg.Type, vmEnv) + inst, reporter, rep, err := bootInstance(mgrcfg) if err != nil { - return fmt.Errorf("failed to create VM pool: %v", err) + return err } - inst, err := vmPool.Create(0) - if err != nil { - return fmt.Errorf("failed to create VM: %v", err) + if rep != nil { + // We should not put rep into resp.CrashTitle/CrashReport, + // because that will be treated as patch not fixing the bug. + return fmt.Errorf("%v\n\n%s\n\n%s", rep.Title, rep.Report, rep.Output) } defer inst.Close() + Logf(0, "job: testing instance...") + rep, err = testInstance(inst, reporter, mgrcfg) + if err != nil { + return err + } + if rep != nil { + // We should not put rep into resp.CrashTitle/CrashReport, + // because that will be treated as patch not fixing the bug. + return fmt.Errorf("%v\n\n%s\n\n%s", rep.Title, rep.Report, rep.Output) + } + Logf(0, "job: copying binaries...") execprogBin, err := inst.Copy(mgrcfg.SyzExecprogBin) if err != nil { @@ -275,11 +285,6 @@ func (job *Job) test() error { if err != nil { return fmt.Errorf("failed to copy to VM: %v", err) } - reporter, err := report.NewReporter(mgrcfg.TargetOS, mgrcfg.Kernel_Src, - filepath.Dir(mgrcfg.Vmlinux), nil, mgrcfg.ParsedIgnores) - if err != nil { - return err - } Logf(0, "job: testing syzkaller program...") opts, err := csource.DeserializeOptions(req.ReproOpts) diff --git a/syz-ci/manager.go b/syz-ci/manager.go index a3a6d1c47..4e4f03485 100644 --- a/syz-ci/manager.go +++ b/syz-ci/manager.go @@ -17,6 +17,7 @@ import ( "github.com/google/syzkaller/pkg/kernel" . "github.com/google/syzkaller/pkg/log" "github.com/google/syzkaller/pkg/osutil" + "github.com/google/syzkaller/pkg/report" "github.com/google/syzkaller/syz-manager/mgrconfig" ) @@ -42,19 +43,20 @@ var imageFiles = []string{ // - latest: latest known good kernel build // - current: currently used kernel build type Manager struct { - name string - workDir string - kernelDir string - currentDir string - latestDir string - compilerID string - configTag string - cfg *Config - mgrcfg *ManagerConfig - managercfg *mgrconfig.Config - cmd *ManagerCmd - dash *dashapi.Dashboard - stop chan struct{} + name string + workDir string + kernelDir string + currentDir string + latestDir string + compilerID string + syzkallerCommit string + configTag string + cfg *Config + mgrcfg *ManagerConfig + managercfg *mgrconfig.Config + cmd *ManagerCmd + dash *dashapi.Dashboard + stop chan struct{} } func createManager(cfg *Config, mgrcfg *ManagerConfig, stop chan struct{}) *Manager { @@ -62,6 +64,9 @@ func createManager(cfg *Config, mgrcfg *ManagerConfig, stop chan struct{}) *Mana if err := osutil.MkdirAll(dir); err != nil { Fatal(err) } + if mgrcfg.Repo_Alias == "" { + mgrcfg.Repo_Alias = mgrcfg.Repo + } var dash *dashapi.Dashboard if cfg.Dashboard_Addr != "" && mgrcfg.Dashboard_Client != "" { @@ -77,6 +82,10 @@ func createManager(cfg *Config, mgrcfg *ManagerConfig, stop chan struct{}) *Mana if err != nil { Fatal(err) } + syzkallerCommit, _ := readTag(filepath.FromSlash("syzkaller/current/tag")) + if syzkallerCommit == "" { + Fatalf("no tag in syzkaller/current/tag") + } // Prepare manager config skeleton (other fields are filled in writeConfig). managercfg := mgrconfig.DefaultValues() @@ -90,18 +99,19 @@ func createManager(cfg *Config, mgrcfg *ManagerConfig, stop chan struct{}) *Mana managercfg.Name = cfg.Name + "-" + mgrcfg.Name mgr := &Manager{ - name: managercfg.Name, - workDir: filepath.Join(dir, "workdir"), - kernelDir: filepath.Join(dir, "kernel"), - currentDir: filepath.Join(dir, "current"), - latestDir: filepath.Join(dir, "latest"), - compilerID: compilerID, - configTag: hash.String(configData), - cfg: cfg, - mgrcfg: mgrcfg, - managercfg: managercfg, - dash: dash, - stop: stop, + name: managercfg.Name, + workDir: filepath.Join(dir, "workdir"), + kernelDir: filepath.Join(dir, "kernel"), + currentDir: filepath.Join(dir, "current"), + latestDir: filepath.Join(dir, "latest"), + compilerID: compilerID, + syzkallerCommit: syzkallerCommit, + configTag: hash.String(configData), + cfg: cfg, + mgrcfg: mgrcfg, + managercfg: managercfg, + dash: dash, + stop: stop, } os.RemoveAll(mgr.currentDir) return mgr @@ -227,8 +237,19 @@ func (mgr *Manager) build() error { if err != nil { return fmt.Errorf("failed to get git HEAD commit: %v", err) } - if err := kernel.Build(mgr.kernelDir, mgr.mgrcfg.Compiler, mgr.mgrcfg.Kernel_Config); err != nil { - return fmt.Errorf("kernel build failed: %v", err) + + var tagData []byte + tagData = append(tagData, kernelCommit...) + tagData = append(tagData, mgr.compilerID...) + tagData = append(tagData, mgr.configTag...) + info := &BuildInfo{ + Time: time.Now(), + Tag: hash.String(tagData), + CompilerID: mgr.compilerID, + KernelRepo: mgr.mgrcfg.Repo, + KernelBranch: mgr.mgrcfg.Branch, + KernelCommit: kernelCommit, + KernelConfigTag: mgr.configTag, } // We first form the whole image in tmp dir and then rename it to latest. @@ -239,6 +260,24 @@ func (mgr *Manager) build() error { if err := osutil.MkdirAll(tmpDir); err != nil { return fmt.Errorf("failed to create tmp dir: %v", err) } + kernelConfig := filepath.Join(tmpDir, "kernel.config") + if err := osutil.CopyFile(mgr.mgrcfg.Kernel_Config, kernelConfig); err != nil { + return err + } + if err := config.SaveFile(filepath.Join(tmpDir, "tag"), info); err != nil { + return fmt.Errorf("failed to write tag file: %v", err) + } + + if err := kernel.Build(mgr.kernelDir, mgr.mgrcfg.Compiler, kernelConfig); err != nil { + rep := &report.Report{ + Title: fmt.Sprintf("%v build error", mgr.mgrcfg.Repo_Alias), + Output: []byte(err.Error()), + } + if err := mgr.reportBuildError(rep, info, tmpDir); err != nil { + Logf(0, "%v: failed to report image error: %v", mgr.name, err) + } + return fmt.Errorf("kernel build failed: %v", err) + } image := filepath.Join(tmpDir, "image") key := filepath.Join(tmpDir, "key") @@ -247,7 +286,6 @@ func (mgr *Manager) build() error { if err != nil { return fmt.Errorf("image build failed: %v", err) } - // TODO(dvyukov): test that the image is good (boots and we can ssh into it). vmlinux := filepath.Join(mgr.kernelDir, "vmlinux") objDir := filepath.Join(tmpDir, "obj") @@ -255,26 +293,9 @@ func (mgr *Manager) build() error { if err := os.Rename(vmlinux, filepath.Join(objDir, "vmlinux")); err != nil { return fmt.Errorf("failed to rename vmlinux file: %v", err) } - kernelConfig := filepath.Join(tmpDir, "kernel.config") - if err := osutil.CopyFile(mgr.mgrcfg.Kernel_Config, kernelConfig); err != nil { - return err - } - var tagData []byte - tagData = append(tagData, kernelCommit...) - tagData = append(tagData, mgr.compilerID...) - tagData = append(tagData, mgr.configTag...) - info := &BuildInfo{ - Time: time.Now(), - Tag: hash.String(tagData), - CompilerID: mgr.compilerID, - KernelRepo: mgr.mgrcfg.Repo, - KernelBranch: mgr.mgrcfg.Branch, - KernelCommit: kernelCommit, - KernelConfigTag: mgr.configTag, - } - if err := config.SaveFile(filepath.Join(tmpDir, "tag"), info); err != nil { - return fmt.Errorf("failed to write tag file: %v", err) + if err := mgr.testImage(tmpDir, info); err != nil { + return err } // Now try to replace latest with our tmp dir as atomically as we can get on Linux. @@ -307,7 +328,7 @@ func (mgr *Manager) restartManager() { Logf(0, "%v: failed to create manager config: %v", mgr.name, err) return } - if err := mgr.uploadBuild(info); err != nil { + if err := mgr.uploadBuild(info, mgr.currentDir); err != nil { Logf(0, "%v: failed to upload build: %v", mgr.name, err) return } @@ -316,6 +337,95 @@ func (mgr *Manager) restartManager() { mgr.cmd = NewManagerCmd(mgr.name, logFile, bin, "-config", cfgFile) } +func (mgr *Manager) testImage(imageDir string, info *BuildInfo) error { + Logf(0, "%v: testing image...", mgr.name) + mgrcfg, err := mgr.createTestConfig(imageDir, info) + if err != nil { + return fmt.Errorf("failed to create manager config: %v", err) + } + switch typ := mgrcfg.Type; typ { + case "gce", "qemu": + default: + // Other types don't support creating machines out of thin air. + return nil + } + if err := osutil.MkdirAll(mgrcfg.Workdir); err != nil { + return fmt.Errorf("failed to create tmp dir: %v", err) + } + defer os.RemoveAll(mgrcfg.Workdir) + + inst, reporter, rep, err := bootInstance(mgrcfg) + if err != nil { + return err + } + if rep != nil { + rep.Title = fmt.Sprintf("%v boot error: %v", mgr.mgrcfg.Repo_Alias, rep.Title) + if err := mgr.reportBuildError(rep, info, imageDir); err != nil { + Logf(0, "%v: failed to report image error: %v", mgr.name, err) + } + return fmt.Errorf("VM boot failed with: %v", rep.Title) + } + defer inst.Close() + rep, err = testInstance(inst, reporter, mgrcfg) + if err != nil { + return err + } + if rep != nil { + rep.Title = fmt.Sprintf("%v test error: %v", mgr.mgrcfg.Repo_Alias, rep.Title) + if err := mgr.reportBuildError(rep, info, imageDir); err != nil { + Logf(0, "%v: failed to report image error: %v", mgr.name, err) + } + return fmt.Errorf("VM testing failed with: %v", rep.Title) + } + return nil +} + +func (mgr *Manager) reportBuildError(rep *report.Report, info *BuildInfo, imageDir string) error { + if mgr.dash == nil { + Logf(0, "%v: image testing failed: %v\n\n%s\n\n%s\n", + mgr.name, rep.Title, rep.Report, rep.Output) + return nil + } + build, err := mgr.createDashboardBuild(info, imageDir) + if err != nil { + return err + } + build.ID += "-error" // must not match normal build ID + req := &dashapi.BuildErrorReq{ + Build: *build, + Crash: dashapi.Crash{ + Title: rep.Title, + Corrupted: rep.Corrupted, + Maintainers: rep.Maintainers, + Log: rep.Output, + Report: rep.Report, + }, + } + return mgr.dash.ReportBuildError(req) +} + +func (mgr *Manager) createTestConfig(imageDir string, info *BuildInfo) (*mgrconfig.Config, error) { + mgrcfg := new(mgrconfig.Config) + *mgrcfg = *mgr.managercfg + mgrcfg.Name += "-test" + mgrcfg.Tag = info.KernelCommit + mgrcfg.Workdir = filepath.Join(imageDir, "workdir") + mgrcfg.Vmlinux = filepath.Join(imageDir, "obj", "vmlinux") + mgrcfg.Image = filepath.Join(imageDir, "image") + mgrcfg.Sshkey = filepath.Join(imageDir, "key") + mgrcfg.Kernel_Src = mgr.kernelDir + mgrcfg.Syzkaller = filepath.FromSlash("syzkaller/current") + cfgdata, err := config.SaveData(mgrcfg) + if err != nil { + return nil, fmt.Errorf("failed to save manager config: %v", err) + } + mgrcfg, err = mgrconfig.LoadData(cfgdata) + if err != nil { + return nil, fmt.Errorf("failed to reload manager config: %v", err) + } + return mgrcfg, nil +} + func (mgr *Manager) writeConfig(info *BuildInfo) (string, error) { mgrcfg := new(mgrconfig.Config) *mgrcfg = *mgr.managercfg @@ -360,39 +470,43 @@ func (mgr *Manager) writeConfig(info *BuildInfo) (string, error) { return configFile, nil } -func (mgr *Manager) uploadBuild(info *BuildInfo) error { +func (mgr *Manager) uploadBuild(info *BuildInfo, imageDir string) error { if mgr.dash == nil { return nil } - syzkallerCommit, _ := readTag(filepath.FromSlash("syzkaller/current/tag")) - if syzkallerCommit == "" { - return fmt.Errorf("no tag in syzkaller/current/tag") - } - kernelConfig, err := ioutil.ReadFile(filepath.Join(mgr.currentDir, "kernel.config")) + build, err := mgr.createDashboardBuild(info, imageDir) if err != nil { - return fmt.Errorf("failed to read kernel.config: %v", err) + return err } commits, err := mgr.pollCommits(info.KernelCommit) if err != nil { // This is not critical for operation. Logf(0, "%v: failed to poll commits: %v", mgr.name, err) } + build.Commits = commits + return mgr.dash.UploadBuild(build) +} + +func (mgr *Manager) createDashboardBuild(info *BuildInfo, imageDir string) (*dashapi.Build, error) { + kernelConfig, err := ioutil.ReadFile(filepath.Join(imageDir, "kernel.config")) + if err != nil { + return nil, fmt.Errorf("failed to read kernel.config: %v", err) + } build := &dashapi.Build{ Manager: mgr.name, ID: info.Tag, OS: mgr.managercfg.TargetOS, Arch: mgr.managercfg.TargetArch, VMArch: mgr.managercfg.TargetVMArch, - SyzkallerCommit: syzkallerCommit, + SyzkallerCommit: mgr.syzkallerCommit, CompilerID: info.CompilerID, KernelRepo: info.KernelRepo, KernelBranch: info.KernelBranch, KernelCommit: info.KernelCommit, KernelConfig: kernelConfig, - Commits: commits, } - return mgr.dash.UploadBuild(build) + return build, nil } // pollCommits asks dashboard what commits it is interested in (i.e. fixes for diff --git a/syz-ci/syz-ci.go b/syz-ci/syz-ci.go index 572df8ac7..94eec3427 100644 --- a/syz-ci/syz-ci.go +++ b/syz-ci/syz-ci.go @@ -87,6 +87,7 @@ type ManagerConfig struct { Dashboard_Client string Dashboard_Key string Repo string + Repo_Alias string // Short name of the repo (e.g. "linux-next"), used only for reporting. Branch string Compiler string Userspace string diff --git a/syz-ci/testing.go b/syz-ci/testing.go new file mode 100644 index 000000000..dcee197ac --- /dev/null +++ b/syz-ci/testing.go @@ -0,0 +1,103 @@ +// Copyright 2017 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "net" + "path/filepath" + "sync/atomic" + "time" + + . "github.com/google/syzkaller/pkg/log" + "github.com/google/syzkaller/pkg/report" + "github.com/google/syzkaller/syz-manager/mgrconfig" + "github.com/google/syzkaller/vm" +) + +// bootInstance boots one VM using the provided config. +// Returns either instance and reporter, or report with boot failure, or error. +func bootInstance(mgrcfg *mgrconfig.Config) (*vm.Instance, report.Reporter, *report.Report, error) { + reporter, err := report.NewReporter(mgrcfg.TargetOS, mgrcfg.Kernel_Src, + filepath.Dir(mgrcfg.Vmlinux), nil, mgrcfg.ParsedIgnores) + if err != nil { + return nil, nil, nil, err + } + vmEnv := mgrconfig.CreateVMEnv(mgrcfg, false) + vmPool, err := vm.Create(mgrcfg.Type, vmEnv) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to create VM pool: %v", err) + } + inst, err := vmPool.Create(0) + if err != nil { + if bootErr, ok := err.(vm.BootError); ok { + rep := reporter.Parse(bootErr.Output) + if rep == nil { + rep = &report.Report{ + Title: bootErr.Title, + Output: bootErr.Output, + } + } + if err := reporter.Symbolize(rep); err != nil { + // TODO(dvyukov): send such errors to dashboard. + Logf(0, "failed to symbolize report: %v", err) + } + return nil, nil, rep, nil + } + return nil, nil, nil, fmt.Errorf("failed to create VM: %v", err) + } + return inst, reporter, nil, nil +} + +// testInstance tests basic operation of the provided VM +// (that we can copy binaries, run binaries, they can connect to host, run syzkaller programs, etc). +// It either returns crash report if there is a kernel bug, +// or err if there is an internal problem, or all nil's if testing succeeded. +func testInstance(inst *vm.Instance, reporter report.Reporter, mgrcfg *mgrconfig.Config) ( + *report.Report, error) { + ln, err := net.Listen("tcp", ":") + if err != nil { + return nil, fmt.Errorf("failed to open listening socket: %v", err) + } + defer ln.Close() + var gotConn uint32 + go func() { + conn, err := ln.Accept() + if err == nil { + conn.Close() + atomic.StoreUint32(&gotConn, 1) + } + return + }() + fwdAddr, err := inst.Forward(ln.Addr().(*net.TCPAddr).Port) + if err != nil { + return nil, fmt.Errorf("failed to setup port forwarding: %v", err) + } + fuzzerBin, err := inst.Copy(mgrcfg.SyzFuzzerBin) + if err != nil { + return nil, fmt.Errorf("failed to copy test binary to VM: %v", err) + } + executorBin, err := inst.Copy(mgrcfg.SyzExecutorBin) + if err != nil { + return nil, fmt.Errorf("failed to copy test binary to VM: %v", err) + } + cmd := fmt.Sprintf("%v -test -executor=%v -name=test -arch=%v -manager=%v -cover=%v -sandbox=%v", + fuzzerBin, executorBin, mgrcfg.TargetArch, fwdAddr, mgrcfg.Cover, mgrcfg.Sandbox) + outc, errc, err := inst.Run(5*time.Minute, nil, cmd) + if err != nil { + return nil, fmt.Errorf("failed to run binary in VM: %v", err) + } + rep := vm.MonitorExecution(outc, errc, reporter, true) + if rep != nil { + if err := reporter.Symbolize(rep); err != nil { + // TODO(dvyukov): send such errors to dashboard. + Logf(0, "failed to symbolize report: %v", err) + } + return rep, nil + } + if atomic.LoadUint32(&gotConn) == 0 { + return nil, fmt.Errorf("test machine failed to connect to host") + } + return nil, nil +} diff --git a/syz-fuzzer/fuzzer.go b/syz-fuzzer/fuzzer.go index ea7b1fcff..3b033b367 100644 --- a/syz-fuzzer/fuzzer.go +++ b/syz-fuzzer/fuzzer.go @@ -40,6 +40,7 @@ var ( flagLeak = flag.Bool("leak", false, "detect memory leaks") flagOutput = flag.String("output", "stdout", "write programs to none/stdout/dmesg/file") flagPprof = flag.String("pprof", "", "address to serve pprof profiles") + flagTest = flag.Bool("test", false, "enable image testing mode") // used by syz-ci ) const ( @@ -121,6 +122,11 @@ func main() { os.Exit(1) }() + if *flagTest { + testImage(*flagManager, target) + return + } + if *flagPprof != "" { go func() { err := http.ListenAndServe(*flagPprof, nil) @@ -223,10 +229,8 @@ func main() { if err != nil { panic(err) } - if _, ok := calls[target.SyscallMap["syz_emit_ethernet"]]; ok { - config.Flags |= ipc.FlagEnableTun - } - if _, ok := calls[target.SyscallMap["syz_extract_tcp_res"]]; ok { + if calls[target.SyscallMap["syz_emit_ethernet"]] || + calls[target.SyscallMap["syz_extract_tcp_res"]] { config.Flags |= ipc.FlagEnableTun } if faultInjectionEnabled { diff --git a/syz-fuzzer/testing.go b/syz-fuzzer/testing.go new file mode 100644 index 000000000..c3ebf9f85 --- /dev/null +++ b/syz-fuzzer/testing.go @@ -0,0 +1,72 @@ +// Copyright 2017 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "net" + + "github.com/google/syzkaller/pkg/host" + "github.com/google/syzkaller/pkg/ipc" + . "github.com/google/syzkaller/pkg/log" + "github.com/google/syzkaller/pkg/osutil" + "github.com/google/syzkaller/prog" +) + +func testImage(hostAddr string, target *prog.Target) { + Logf(0, "connecting to host at %v", hostAddr) + conn, err := net.Dial("tcp", hostAddr) + if err != nil { + Fatalf("failed to connect: %v", err) + } + conn.Close() + + Logf(0, "checking config...") + config, err := ipc.DefaultConfig() + if err != nil { + Fatalf("failed to create ipc config: %v", err) + } + if kcov, _ := checkCompsSupported(); !kcov && config.Flags&ipc.FlagSignal != 0 { + Fatalf("coverage is not supported by kernel") + } + if config.Flags&ipc.FlagSandboxNamespace != 0 && !osutil.IsExist("/proc/self/ns/user") { + Fatalf("/proc/self/ns/user is not present for namespace sandbox") + } + calls, err := host.DetectSupportedSyscalls(target) + if err != nil { + Fatalf("failed to detect supported syscalls: %v", err) + } + calls = target.TransitivelyEnabledCalls(calls) + Logf(0, "enabled syscalls: %v", len(calls)) + if calls[target.SyscallMap["syz_emit_ethernet"]] || + calls[target.SyscallMap["syz_extract_tcp_res"]] { + config.Flags |= ipc.FlagEnableTun + } + + Logf(0, "testing simple program...") + env, err := ipc.MakeEnv(*flagExecutor, 0, config) + if err != nil { + Fatalf("failed to create ipc env: %v", err) + } + p := target.GenerateSimpleProg() + opts := &ipc.ExecOpts{} + output, info, failed, hanged, err := env.Exec(opts, p) + if err != nil { + Fatalf("execution failed: %v\n%s", err, output) + } + if hanged { + Fatalf("program hanged:\n%s", output) + } + if failed { + Fatalf("program failed:\n%s", output) + } + if len(info) == 0 { + Fatalf("no calls executed:\n%s", output) + } + if info[0].Errno != 0 { + Fatalf("simple call failed: %v\n%s", info[0].Errno, output) + } + if config.Flags&ipc.FlagSignal != 0 && len(info[0].Signal) == 0 { + Fatalf("got no coverage:\n%s", output) + } +} diff --git a/vm/gce/gce.go b/vm/gce/gce.go index 96cc14e79..792982560 100644 --- a/vm/gce/gce.go +++ b/vm/gce/gce.go @@ -364,7 +364,7 @@ func (pool *Pool) waitInstanceBoot(name, ip, sshKey, sshUser, gceKey string) err if err != nil { output = []byte(fmt.Sprintf("failed to get boot output: %v", err)) } - return fmt.Errorf("can't ssh into the instance\n\n%s", output) + return vmimpl.BootError{"can't ssh into the instance", output} } func (pool *Pool) getSerialPortOutput(name, gceKey string) ([]byte, error) { diff --git a/vm/qemu/qemu.go b/vm/qemu/qemu.go index 679dcf25e..e3f33554a 100644 --- a/vm/qemu/qemu.go +++ b/vm/qemu/qemu.go @@ -373,13 +373,13 @@ func (inst *instance) Boot() error { time.Sleep(time.Second) // wait for any pending output bootOutputStop <- true <-bootOutputStop - return fmt.Errorf("qemu stopped:\n%v\n", string(bootOutput)) + return vmimpl.BootError{"qemu stopped", bootOutput} default: } if time.Since(start) > 10*time.Minute { bootOutputStop <- true <-bootOutputStop - return fmt.Errorf("ssh server did not start:\n%v\n", string(bootOutput)) + return vmimpl.BootError{"ssh server did not start", bootOutput} } } bootOutputStop <- true diff --git a/vm/vm.go b/vm/vm.go index b59c0d223..48b68d597 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -37,13 +37,20 @@ type Instance struct { index int } -type Env vmimpl.Env +type ( + Env vmimpl.Env + BootError vmimpl.BootError +) var ( Shutdown = vmimpl.Shutdown TimeoutErr = vmimpl.TimeoutErr ) +func (err BootError) Error() string { + return fmt.Sprintf("%v\n%s", err.Title, err.Output) +} + func Create(typ string, env *Env) (*Pool, error) { impl, err := vmimpl.Create(typ, (*vmimpl.Env)(env)) if err != nil { diff --git a/vm/vmimpl/vmimpl.go b/vm/vmimpl/vmimpl.go index 617f9bc0f..81f798d26 100644 --- a/vm/vmimpl/vmimpl.go +++ b/vm/vmimpl/vmimpl.go @@ -56,6 +56,16 @@ type Env struct { Config []byte // json-serialized VM-type-specific config } +// BootError is returned by Pool.Create when VM does not boot. +type BootError struct { + Title string + Output []byte +} + +func (err BootError) Error() string { + return fmt.Sprintf("%v\n%s", err.Title, err.Output) +} + // Create creates a VM type that can be used to create individual VMs. func Create(typ string, env *Env) (Pool, error) { ctor := ctors[typ] -- cgit mrf-deployment