syz-ci: test images before using them

Boot and minimally test images before declaring them as good
and switching to using them.

If image build/boot/test fails, upload report about this to dashboard.

20 files changed, 489 insertions, 107 deletions

diff --git a/dashboard/app/api.go b/dashboard/app/api.go
index a3a466ce6..db265f8b8 100644
--- a/dashboard/app/api.go
+++ b/dashboard/app/api.go
@@ -29,20 +29,25 @@ func init() {
 }
 
 var apiHandlers = map[string]APIHandler{
-	"log_error":           apiLogError,
+	"log_error":        apiLogError,
+	"job_poll":         apiJobPoll,
+	"job_done":         apiJobDone,
+	"reporting_poll":   apiReportingPoll,
+	"reporting_update": apiReportingUpdate,
+}
+
+var apiNamespaceHandlers = map[string]APINamespaceHandler{
 	"upload_build":        apiUploadBuild,
 	"builder_poll":        apiBuilderPoll,
-	"job_poll":            apiJobPoll,
-	"job_done":            apiJobDone,
+	"report_build_error":  apiReportBuildError,
 	"report_crash":        apiReportCrash,
 	"report_failed_repro": apiReportFailedRepro,
 	"need_repro":          apiNeedRepro,
-	"reporting_poll":      apiReportingPoll,
-	"reporting_update":    apiReportingUpdate,
 }
 
 type JSONHandler func(c context.Context, r *http.Request) (interface{}, error)
-type APIHandler func(c context.Context, ns string, r *http.Request) (interface{}, error)
+type APIHandler func(c context.Context, r *http.Request) (interface{}, error)
+type APINamespaceHandler func(c context.Context, ns string, r *http.Request) (interface{}, error)
 
 // Overridable for testing.
 var timeNow = func(c context.Context) time.Time {
@@ -82,10 +87,17 @@ func handleAPI(c context.Context, r *http.Request) (reply interface{}, err error
 	}
 	method := r.FormValue("method")
 	handler := apiHandlers[method]
-	if handler == nil {
+	if handler != nil {
+		return handler(c, r)
+	}
+	nsHandler := apiNamespaceHandlers[method]
+	if nsHandler == nil {
 		return nil, fmt.Errorf("unknown api method %q", method)
 	}
-	return handler(c, ns, r)
+	if ns == "" {
+		return nil, fmt.Errorf("method %q must be called within a namespace", method)
+	}
+	return nsHandler(c, ns, r)
 }
 
 func checkClient(c context.Context, name0, key0 string) (string, error) {
@@ -110,7 +122,7 @@ func checkClient(c context.Context, name0, key0 string) (string, error) {
 	return "", fmt.Errorf("unauthorized api request from %q", name0)
 }
 
-func apiLogError(c context.Context, ns string, r *http.Request) (interface{}, error) {
+func apiLogError(c context.Context, r *http.Request) (interface{}, error) {
 	req := new(dashapi.LogEntry)
 	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %v", err)
@@ -159,7 +171,7 @@ loop:
 	return resp, nil
 }
 
-func apiJobPoll(c context.Context, ns string, r *http.Request) (interface{}, error) {
+func apiJobPoll(c context.Context, r *http.Request) (interface{}, error) {
 	req := new(dashapi.JobPollReq)
 	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %v", err)
@@ -170,7 +182,7 @@ func apiJobPoll(c context.Context, ns string, r *http.Request) (interface{}, err
 	return pollPendingJobs(c, req.Managers)
 }
 
-func apiJobDone(c context.Context, ns string, r *http.Request) (interface{}, error) {
+func apiJobDone(c context.Context, r *http.Request) (interface{}, error) {
 	req := new(dashapi.JobDoneReq)
 	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %v", err)
@@ -184,11 +196,22 @@ func apiUploadBuild(c context.Context, ns string, r *http.Request) (interface{},
 	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %v", err)
 	}
-	err := uploadBuild(c, ns, req)
-	return nil, err
+	if err := uploadBuild(c, ns, req, BuildNormal); err != nil {
+		return nil, err
+	}
+	if len(req.Commits) != 0 {
+		if err := addCommitsToBugs(c, ns, req.Manager, req.Commits); err != nil {
+			return nil, err
+		}
+	}
+	return nil, nil
 }
 
-func uploadBuild(c context.Context, ns string, req *dashapi.Build) error {
+func uploadBuild(c context.Context, ns string, req *dashapi.Build, typ BuildType) error {
+	if _, err := loadBuild(c, ns, req.ID); err == nil {
+		return nil
+	}
+
 	checkStrLen := func(str, name string, maxLen int) error {
 		if str == "" {
 			return fmt.Errorf("%v is empty", name)
@@ -227,6 +250,8 @@ func uploadBuild(c context.Context, ns string, req *dashapi.Build) error {
 		Namespace:       ns,
 		Manager:         req.Manager,
 		ID:              req.ID,
+		Type:            typ,
+		Time:            timeNow(c),
 		OS:              req.OS,
 		Arch:            req.Arch,
 		VMArch:          req.VMArch,
@@ -240,13 +265,6 @@ func uploadBuild(c context.Context, ns string, req *dashapi.Build) error {
 	if _, err := datastore.Put(c, buildKey(c, ns, req.ID), build); err != nil {
 		return err
 	}
-
-	if len(req.Commits) != 0 {
-		if err := addCommitsToBugs(c, ns, req.Manager, req.Commits); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
@@ -348,6 +366,20 @@ func stringInList(list []string, str string) bool {
 	return false
 }
 
+func apiReportBuildError(c context.Context, ns string, r *http.Request) (interface{}, error) {
+	req := new(dashapi.BuildErrorReq)
+	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal request: %v", err)
+	}
+	if err := uploadBuild(c, ns, &req.Build, BuildFailed); err != nil {
+		return nil, err
+	}
+	if _, err := reportCrash(c, ns, &req.Crash); err != nil {
+		return nil, err
+	}
+	return nil, nil
+}
+
 const corruptedReportTitle = "corrupted report"
 
 func apiReportCrash(c context.Context, ns string, r *http.Request) (interface{}, error) {
@@ -355,6 +387,10 @@ func apiReportCrash(c context.Context, ns string, r *http.Request) (interface{},
 	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %v", err)
 	}
+	return reportCrash(c, ns, req)
+}
+
+func reportCrash(c context.Context, ns string, req *dashapi.Crash) (interface{}, error) {
 	req.Title = limitLength(req.Title, maxTextLen)
 	req.Maintainers = email.MergeEmailLists(req.Maintainers)
 	if req.Corrupted {
diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go
index 6c63ef5e3..6661b84b7 100644
--- a/dashboard/app/entities.go
+++ b/dashboard/app/entities.go
@@ -28,6 +28,8 @@ type Build struct {
 	Namespace       string
 	Manager         string
 	ID              string // unique ID generated by syz-ci
+	Type            BuildType
+	Time            time.Time
 	OS              string
 	Arch            string
 	VMArch          string
@@ -153,6 +155,14 @@ const (
 	ReproLevelC    = dashapi.ReproLevelC
 )
 
+type BuildType int
+
+const (
+	BuildNormal BuildType = iota
+	BuildFailed
+	BuildJob
+)
+
 func buildKey(c context.Context, ns, id string) *datastore.Key {
 	if ns == "" {
 		panic("requesting build key outside of namespace")
diff --git a/dashboard/app/jobs.go b/dashboard/app/jobs.go
index 94f1400d8..42743c78d 100644
--- a/dashboard/app/jobs.go
+++ b/dashboard/app/jobs.go
@@ -244,7 +244,7 @@ func doneJob(c context.Context, req *dashapi.JobDoneReq) error {
 			return fmt.Errorf("job %v: already finished", jobID)
 		}
 		ns := job.Namespace
-		if err := uploadBuild(c, ns, &req.Build); err != nil {
+		if err := uploadBuild(c, ns, &req.Build, BuildJob); err != nil {
 			return err
 		}
 		if job.Error, err = putText(c, ns, "Error", req.Error, false); err != nil {
diff --git a/dashboard/app/jobs_test.go b/dashboard/app/jobs_test.go
index af997f69b..f8d82ee22 100644
--- a/dashboard/app/jobs_test.go
+++ b/dashboard/app/jobs_test.go
@@ -145,7 +145,7 @@ Raw console output is attached.
 		c.expectEQ(msg.Attachments[1].Data, []byte(patch))
 		c.expectEQ(msg.Body, `Hello,
 
-syzbot tried to test the proposed patch but build failed:
+syzbot tried to test the proposed patch but build/boot failed:
 
 failed to apply patch
 
diff --git a/dashboard/app/mail_test_result.txt b/dashboard/app/mail_test_result.txt
index 83009298a..9219d1298 100644
--- a/dashboard/app/mail_test_result.txt
+++ b/dashboard/app/mail_test_result.txt
@@ -5,7 +5,7 @@ syzbot has tested the proposed patch but the reproducer still triggered crash:
 
 {{printf "%s" .Report}}
 {{else if .Error}}
-syzbot tried to test the proposed patch but build failed:
+syzbot tried to test the proposed patch but build/boot failed:
 
 {{printf "%s" .Error}}
 {{if .ErrorTruncated}}
diff --git a/dashboard/app/reporting_external.go b/dashboard/app/reporting_external.go
index 56f8c3b4c..c5e7c06fa 100644
--- a/dashboard/app/reporting_external.go
+++ b/dashboard/app/reporting_external.go
@@ -24,7 +24,7 @@ func (cfg *ExternalConfig) Type() string {
 	return cfg.ID
 }
 
-func apiReportingPoll(c context.Context, ns string, r *http.Request) (interface{}, error) {
+func apiReportingPoll(c context.Context, r *http.Request) (interface{}, error) {
 	req := new(dashapi.PollRequest)
 	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %v", err)
@@ -36,7 +36,7 @@ func apiReportingPoll(c context.Context, ns string, r *http.Request) (interface{
 	return resp, nil
 }
 
-func apiReportingUpdate(c context.Context, ns string, r *http.Request) (interface{}, error) {
+func apiReportingUpdate(c context.Context, r *http.Request) (interface{}, error) {
 	req := new(dashapi.BugUpdate)
 	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %v", err)
diff --git a/dashboard/dashapi/dashapi.go b/dashboard/dashapi/dashapi.go
index 50c144dae..8ebea29ac 100644
--- a/dashboard/dashapi/dashapi.go
+++ b/dashboard/dashapi/dashapi.go
@@ -121,6 +121,15 @@ func (dash *Dashboard) JobDone(req *JobDoneReq) error {
 	return dash.query("job_done", req, nil)
 }
 
+type BuildErrorReq struct {
+	Build Build
+	Crash Crash
+}
+
+func (dash *Dashboard) ReportBuildError(req *BuildErrorReq) error {
+	return dash.query("report_build_error", req, nil)
+}
+
 // Crash describes a single kernel crash (potentially with repro).
 type Crash struct {
 	BuildID     string // refers to Build.ID
diff --git a/pkg/ipc/ipc.go b/pkg/ipc/ipc.go
index 2f1cf3ef1..f6576342f 100644
--- a/pkg/ipc/ipc.go
+++ b/pkg/ipc/ipc.go
@@ -308,8 +308,7 @@ func (env *Env) Exec(opts *ExecOpts, p *prog.Prog) (output []byte, info []CallIn
 	if env.config.Flags&FlagUseShmem == 0 {
 		progData = env.in[:progSize]
 	}
-	needOutput := env.config.Flags&FlagSignal != 0 || opts.Flags&FlagCollectComps != 0
-	if needOutput && env.out != nil {
+	if env.out != nil {
 		// Zero out the first two words (ncmd and nsig), so that we don't have garbage there
 		// if executor crashes before writing non-garbage there.
 		for i := 0; i < 4; i++ {
@@ -333,7 +332,7 @@ func (env *Env) Exec(opts *ExecOpts, p *prog.Prog) (output []byte, info []CallIn
 		return
 	}
 
-	if needOutput && env.out != nil {
+	if env.out != nil {
 		info, err0 = env.readOutCoverage(p)
 	}
 	return
diff --git a/pkg/ipc/ipc_test.go b/pkg/ipc/ipc_test.go
index 51e961119..6d1c2c80e 100644
--- a/pkg/ipc/ipc_test.go
+++ b/pkg/ipc/ipc_test.go
@@ -114,6 +114,9 @@ func TestExecute(t *testing.T) {
 
 		for i := 0; i < iters/len(flags); i++ {
 			p := target.Generate(rs, 10, nil)
+			if i == 0 {
+				p = target.GenerateSimpleProg()
+			}
 			opts := &ExecOpts{}
 			output, _, _, _, err := env.Exec(opts, p)
 			if err != nil {
diff --git a/prog/rand.go b/prog/rand.go
index 2b7709907..839c6d1c3 100644
--- a/prog/rand.go
+++ b/prog/rand.go
@@ -499,6 +499,15 @@ func (target *Target) GenerateAllSyzProg(rs rand.Source) *Prog {
 	return p
 }
 
+// GenerateSimpleProg generates the simplest non-empty program for testing
+// (e.g. containing a single mmap).
+func (target *Target) GenerateSimpleProg() *Prog {
+	return &Prog{
+		Target: target,
+		Calls:  []*Call{target.MakeMmap(0, 1)},
+	}
+}
+
 func (r *randGen) generateArgs(s *state, types []Type) ([]Arg, []*Call) {
 	var calls []*Call
 	args := make([]Arg, len(types))
diff --git a/syz-ci/jobs.go b/syz-ci/jobs.go
index 1a7eb14af..604dd1895 100644
--- a/syz-ci/jobs.go
+++ b/syz-ci/jobs.go
@@ -220,7 +220,6 @@ func (job *Job) buildImage() error {
 	if err != nil {
 		return fmt.Errorf("image build failed: %v", err)
 	}
-	// TODO(dvyukov): test that the image is good (boots and we can ssh into it).
 
 	mgrcfg := new(mgrconfig.Config)
 	*mgrcfg = *mgr.managercfg
@@ -247,17 +246,28 @@ func (job *Job) test() error {
 	req, mgrcfg := job.req, job.mgrcfg
 
 	Logf(0, "job: booting VM...")
-	vmEnv := mgrconfig.CreateVMEnv(mgrcfg, false)
-	vmPool, err := vm.Create(mgrcfg.Type, vmEnv)
+	inst, reporter, rep, err := bootInstance(mgrcfg)
 	if err != nil {
-		return fmt.Errorf("failed to create VM pool: %v", err)
+		return err
 	}
-	inst, err := vmPool.Create(0)
-	if err != nil {
-		return fmt.Errorf("failed to create VM: %v", err)
+	if rep != nil {
+		// We should not put rep into resp.CrashTitle/CrashReport,
+		// because that will be treated as patch not fixing the bug.
+		return fmt.Errorf("%v\n\n%s\n\n%s", rep.Title, rep.Report, rep.Output)
 	}
 	defer inst.Close()
 
+	Logf(0, "job: testing instance...")
+	rep, err = testInstance(inst, reporter, mgrcfg)
+	if err != nil {
+		return err
+	}
+	if rep != nil {
+		// We should not put rep into resp.CrashTitle/CrashReport,
+		// because that will be treated as patch not fixing the bug.
+		return fmt.Errorf("%v\n\n%s\n\n%s", rep.Title, rep.Report, rep.Output)
+	}
+
 	Logf(0, "job: copying binaries...")
 	execprogBin, err := inst.Copy(mgrcfg.SyzExecprogBin)
 	if err != nil {
@@ -275,11 +285,6 @@ func (job *Job) test() error {
 	if err != nil {
 		return fmt.Errorf("failed to copy to VM: %v", err)
 	}
-	reporter, err := report.NewReporter(mgrcfg.TargetOS, mgrcfg.Kernel_Src,
-		filepath.Dir(mgrcfg.Vmlinux), nil, mgrcfg.ParsedIgnores)
-	if err != nil {
-		return err
-	}
 
 	Logf(0, "job: testing syzkaller program...")
 	opts, err := csource.DeserializeOptions(req.ReproOpts)
diff --git a/syz-ci/manager.go b/syz-ci/manager.go
index a3a6d1c47..4e4f03485 100644
--- a/syz-ci/manager.go
+++ b/syz-ci/manager.go
@@ -17,6 +17,7 @@ import (
 	"github.com/google/syzkaller/pkg/kernel"
 	. "github.com/google/syzkaller/pkg/log"
 	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/pkg/report"
 	"github.com/google/syzkaller/syz-manager/mgrconfig"
 )
 
@@ -42,19 +43,20 @@ var imageFiles = []string{
 //  - latest: latest known good kernel build
 //  - current: currently used kernel build
 type Manager struct {
-	name       string
-	workDir    string
-	kernelDir  string
-	currentDir string
-	latestDir  string
-	compilerID string
-	configTag  string
-	cfg        *Config
-	mgrcfg     *ManagerConfig
-	managercfg *mgrconfig.Config
-	cmd        *ManagerCmd
-	dash       *dashapi.Dashboard
-	stop       chan struct{}
+	name            string
+	workDir         string
+	kernelDir       string
+	currentDir      string
+	latestDir       string
+	compilerID      string
+	syzkallerCommit string
+	configTag       string
+	cfg             *Config
+	mgrcfg          *ManagerConfig
+	managercfg      *mgrconfig.Config
+	cmd             *ManagerCmd
+	dash            *dashapi.Dashboard
+	stop            chan struct{}
 }
 
 func createManager(cfg *Config, mgrcfg *ManagerConfig, stop chan struct{}) *Manager {
@@ -62,6 +64,9 @@ func createManager(cfg *Config, mgrcfg *ManagerConfig, stop chan struct{}) *Mana
 	if err := osutil.MkdirAll(dir); err != nil {
 		Fatal(err)
 	}
+	if mgrcfg.Repo_Alias == "" {
+		mgrcfg.Repo_Alias = mgrcfg.Repo
+	}
 
 	var dash *dashapi.Dashboard
 	if cfg.Dashboard_Addr != "" && mgrcfg.Dashboard_Client != "" {
@@ -77,6 +82,10 @@ func createManager(cfg *Config, mgrcfg *ManagerConfig, stop chan struct{}) *Mana
 	if err != nil {
 		Fatal(err)
 	}
+	syzkallerCommit, _ := readTag(filepath.FromSlash("syzkaller/current/tag"))
+	if syzkallerCommit == "" {
+		Fatalf("no tag in syzkaller/current/tag")
+	}
 
 	// Prepare manager config skeleton (other fields are filled in writeConfig).
 	managercfg := mgrconfig.DefaultValues()
@@ -90,18 +99,19 @@ func createManager(cfg *Config, mgrcfg *ManagerConfig, stop chan struct{}) *Mana
 	managercfg.Name = cfg.Name + "-" + mgrcfg.Name
 
 	mgr := &Manager{
-		name:       managercfg.Name,
-		workDir:    filepath.Join(dir, "workdir"),
-		kernelDir:  filepath.Join(dir, "kernel"),
-		currentDir: filepath.Join(dir, "current"),
-		latestDir:  filepath.Join(dir, "latest"),
-		compilerID: compilerID,
-		configTag:  hash.String(configData),
-		cfg:        cfg,
-		mgrcfg:     mgrcfg,
-		managercfg: managercfg,
-		dash:       dash,
-		stop:       stop,
+		name:            managercfg.Name,
+		workDir:         filepath.Join(dir, "workdir"),
+		kernelDir:       filepath.Join(dir, "kernel"),
+		currentDir:      filepath.Join(dir, "current"),
+		latestDir:       filepath.Join(dir, "latest"),
+		compilerID:      compilerID,
+		syzkallerCommit: syzkallerCommit,
+		configTag:       hash.String(configData),
+		cfg:             cfg,
+		mgrcfg:          mgrcfg,
+		managercfg:      managercfg,
+		dash:            dash,
+		stop:            stop,
 	}
 	os.RemoveAll(mgr.currentDir)
 	return mgr
@@ -227,8 +237,19 @@ func (mgr *Manager) build() error {
 	if err != nil {
 		return fmt.Errorf("failed to get git HEAD commit: %v", err)
 	}
-	if err := kernel.Build(mgr.kernelDir, mgr.mgrcfg.Compiler, mgr.mgrcfg.Kernel_Config); err != nil {
-		return fmt.Errorf("kernel build failed: %v", err)
+
+	var tagData []byte
+	tagData = append(tagData, kernelCommit...)
+	tagData = append(tagData, mgr.compilerID...)
+	tagData = append(tagData, mgr.configTag...)
+	info := &BuildInfo{
+		Time:            time.Now(),
+		Tag:             hash.String(tagData),
+		CompilerID:      mgr.compilerID,
+		KernelRepo:      mgr.mgrcfg.Repo,
+		KernelBranch:    mgr.mgrcfg.Branch,
+		KernelCommit:    kernelCommit,
+		KernelConfigTag: mgr.configTag,
 	}
 
 	// We first form the whole image in tmp dir and then rename it to latest.
@@ -239,6 +260,24 @@ func (mgr *Manager) build() error {
 	if err := osutil.MkdirAll(tmpDir); err != nil {
 		return fmt.Errorf("failed to create tmp dir: %v", err)
 	}
+	kernelConfig := filepath.Join(tmpDir, "kernel.config")
+	if err := osutil.CopyFile(mgr.mgrcfg.Kernel_Config, kernelConfig); err != nil {
+		return err
+	}
+	if err := config.SaveFile(filepath.Join(tmpDir, "tag"), info); err != nil {
+		return fmt.Errorf("failed to write tag file: %v", err)
+	}
+
+	if err := kernel.Build(mgr.kernelDir, mgr.mgrcfg.Compiler, kernelConfig); err != nil {
+		rep := &report.Report{
+			Title:  fmt.Sprintf("%v build error", mgr.mgrcfg.Repo_Alias),
+			Output: []byte(err.Error()),
+		}
+		if err := mgr.reportBuildError(rep, info, tmpDir); err != nil {
+			Logf(0, "%v: failed to report image error: %v", mgr.name, err)
+		}
+		return fmt.Errorf("kernel build failed: %v", err)
+	}
 
 	image := filepath.Join(tmpDir, "image")
 	key := filepath.Join(tmpDir, "key")
@@ -247,7 +286,6 @@ func (mgr *Manager) build() error {
 	if err != nil {
 		return fmt.Errorf("image build failed: %v", err)
 	}
-	// TODO(dvyukov): test that the image is good (boots and we can ssh into it).
 
 	vmlinux := filepath.Join(mgr.kernelDir, "vmlinux")
 	objDir := filepath.Join(tmpDir, "obj")
@@ -255,26 +293,9 @@ func (mgr *Manager) build() error {
 	if err := os.Rename(vmlinux, filepath.Join(objDir, "vmlinux")); err != nil {
 		return fmt.Errorf("failed to rename vmlinux file: %v", err)
 	}
-	kernelConfig := filepath.Join(tmpDir, "kernel.config")
-	if err := osutil.CopyFile(mgr.mgrcfg.Kernel_Config, kernelConfig); err != nil {
-		return err
-	}
 
-	var tagData []byte
-	tagData = append(tagData, kernelCommit...)
-	tagData = append(tagData, mgr.compilerID...)
-	tagData = append(tagData, mgr.configTag...)
-	info := &BuildInfo{
-		Time:            time.Now(),
-		Tag:             hash.String(tagData),
-		CompilerID:      mgr.compilerID,
-		KernelRepo:      mgr.mgrcfg.Repo,
-		KernelBranch:    mgr.mgrcfg.Branch,
-		KernelCommit:    kernelCommit,
-		KernelConfigTag: mgr.configTag,
-	}
-	if err := config.SaveFile(filepath.Join(tmpDir, "tag"), info); err != nil {
-		return fmt.Errorf("failed to write tag file: %v", err)
+	if err := mgr.testImage(tmpDir, info); err != nil {
+		return err
 	}
 
 	// Now try to replace latest with our tmp dir as atomically as we can get on Linux.
@@ -307,7 +328,7 @@ func (mgr *Manager) restartManager() {
 		Logf(0, "%v: failed to create manager config: %v", mgr.name, err)
 		return
 	}
-	if err := mgr.uploadBuild(info); err != nil {
+	if err := mgr.uploadBuild(info, mgr.currentDir); err != nil {
 		Logf(0, "%v: failed to upload build: %v", mgr.name, err)
 		return
 	}
@@ -316,6 +337,95 @@ func (mgr *Manager) restartManager() {
 	mgr.cmd = NewManagerCmd(mgr.name, logFile, bin, "-config", cfgFile)
 }
 
+func (mgr *Manager) testImage(imageDir string, info *BuildInfo) error {
+	Logf(0, "%v: testing image...", mgr.name)
+	mgrcfg, err := mgr.createTestConfig(imageDir, info)
+	if err != nil {
+		return fmt.Errorf("failed to create manager config: %v", err)
+	}
+	switch typ := mgrcfg.Type; typ {
+	case "gce", "qemu":
+	default:
+		// Other types don't support creating machines out of thin air.
+		return nil
+	}
+	if err := osutil.MkdirAll(mgrcfg.Workdir); err != nil {
+		return fmt.Errorf("failed to create tmp dir: %v", err)
+	}
+	defer os.RemoveAll(mgrcfg.Workdir)
+
+	inst, reporter, rep, err := bootInstance(mgrcfg)
+	if err != nil {
+		return err
+	}
+	if rep != nil {
+		rep.Title = fmt.Sprintf("%v boot error: %v", mgr.mgrcfg.Repo_Alias, rep.Title)
+		if err := mgr.reportBuildError(rep, info, imageDir); err != nil {
+			Logf(0, "%v: failed to report image error: %v", mgr.name, err)
+		}
+		return fmt.Errorf("VM boot failed with: %v", rep.Title)
+	}
+	defer inst.Close()
+	rep, err = testInstance(inst, reporter, mgrcfg)
+	if err != nil {
+		return err
+	}
+	if rep != nil {
+		rep.Title = fmt.Sprintf("%v test error: %v", mgr.mgrcfg.Repo_Alias, rep.Title)
+		if err := mgr.reportBuildError(rep, info, imageDir); err != nil {
+			Logf(0, "%v: failed to report image error: %v", mgr.name, err)
+		}
+		return fmt.Errorf("VM testing failed with: %v", rep.Title)
+	}
+	return nil
+}
+
+func (mgr *Manager) reportBuildError(rep *report.Report, info *BuildInfo, imageDir string) error {
+	if mgr.dash == nil {
+		Logf(0, "%v: image testing failed: %v\n\n%s\n\n%s\n",
+			mgr.name, rep.Title, rep.Report, rep.Output)
+		return nil
+	}
+	build, err := mgr.createDashboardBuild(info, imageDir)
+	if err != nil {
+		return err
+	}
+	build.ID += "-error" // must not match normal build ID
+	req := &dashapi.BuildErrorReq{
+		Build: *build,
+		Crash: dashapi.Crash{
+			Title:       rep.Title,
+			Corrupted:   rep.Corrupted,
+			Maintainers: rep.Maintainers,
+			Log:         rep.Output,
+			Report:      rep.Report,
+		},
+	}
+	return mgr.dash.ReportBuildError(req)
+}
+
+func (mgr *Manager) createTestConfig(imageDir string, info *BuildInfo) (*mgrconfig.Config, error) {
+	mgrcfg := new(mgrconfig.Config)
+	*mgrcfg = *mgr.managercfg
+	mgrcfg.Name += "-test"
+	mgrcfg.Tag = info.KernelCommit
+	mgrcfg.Workdir = filepath.Join(imageDir, "workdir")
+	mgrcfg.Vmlinux = filepath.Join(imageDir, "obj", "vmlinux")
+	mgrcfg.Image = filepath.Join(imageDir, "image")
+	mgrcfg.Sshkey = filepath.Join(imageDir, "key")
+	mgrcfg.Kernel_Src = mgr.kernelDir
+	mgrcfg.Syzkaller = filepath.FromSlash("syzkaller/current")
+	cfgdata, err := config.SaveData(mgrcfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to save manager config: %v", err)
+	}
+	mgrcfg, err = mgrconfig.LoadData(cfgdata)
+	if err != nil {
+		return nil, fmt.Errorf("failed to reload manager config: %v", err)
+	}
+	return mgrcfg, nil
+}
+
 func (mgr *Manager) writeConfig(info *BuildInfo) (string, error) {
 	mgrcfg := new(mgrconfig.Config)
 	*mgrcfg = *mgr.managercfg
@@ -360,39 +470,43 @@ func (mgr *Manager) writeConfig(info *BuildInfo) (string, error) {
 	return configFile, nil
 }
 
-func (mgr *Manager) uploadBuild(info *BuildInfo) error {
+func (mgr *Manager) uploadBuild(info *BuildInfo, imageDir string) error {
 	if mgr.dash == nil {
 		return nil
 	}
 
-	syzkallerCommit, _ := readTag(filepath.FromSlash("syzkaller/current/tag"))
-	if syzkallerCommit == "" {
-		return fmt.Errorf("no tag in syzkaller/current/tag")
-	}
-	kernelConfig, err := ioutil.ReadFile(filepath.Join(mgr.currentDir, "kernel.config"))
+	build, err := mgr.createDashboardBuild(info, imageDir)
 	if err != nil {
-		return fmt.Errorf("failed to read kernel.config: %v", err)
+		return err
 	}
 	commits, err := mgr.pollCommits(info.KernelCommit)
 	if err != nil {
 		// This is not critical for operation.
 		Logf(0, "%v: failed to poll commits: %v", mgr.name, err)
 	}
+	build.Commits = commits
+	return mgr.dash.UploadBuild(build)
+}
+
+func (mgr *Manager) createDashboardBuild(info *BuildInfo, imageDir string) (*dashapi.Build, error) {
+	kernelConfig, err := ioutil.ReadFile(filepath.Join(imageDir, "kernel.config"))
+	if err != nil {
+		return nil, fmt.Errorf("failed to read kernel.config: %v", err)
+	}
 	build := &dashapi.Build{
 		Manager:         mgr.name,
 		ID:              info.Tag,
 		OS:              mgr.managercfg.TargetOS,
 		Arch:            mgr.managercfg.TargetArch,
 		VMArch:          mgr.managercfg.TargetVMArch,
-		SyzkallerCommit: syzkallerCommit,
+		SyzkallerCommit: mgr.syzkallerCommit,
 		CompilerID:      info.CompilerID,
 		KernelRepo:      info.KernelRepo,
 		KernelBranch:    info.KernelBranch,
 		KernelCommit:    info.KernelCommit,
 		KernelConfig:    kernelConfig,
-		Commits:         commits,
 	}
-	return mgr.dash.UploadBuild(build)
+	return build, nil
 }
 
 // pollCommits asks dashboard what commits it is interested in (i.e. fixes for
diff --git a/syz-ci/syz-ci.go b/syz-ci/syz-ci.go
index 572df8ac7..94eec3427 100644
--- a/syz-ci/syz-ci.go
+++ b/syz-ci/syz-ci.go
@@ -87,6 +87,7 @@ type ManagerConfig struct {
 	Dashboard_Client string
 	Dashboard_Key    string
 	Repo             string
+	Repo_Alias       string // Short name of the repo (e.g. "linux-next"), used only for reporting.
 	Branch           string
 	Compiler         string
 	Userspace        string
diff --git a/syz-ci/testing.go b/syz-ci/testing.go
new file mode 100644
index 000000000..dcee197ac
--- /dev/null
+++ b/syz-ci/testing.go
@@ -0,0 +1,103 @@
+// Copyright 2017 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"net"
+	"path/filepath"
+	"sync/atomic"
+	"time"
+
+	. "github.com/google/syzkaller/pkg/log"
+	"github.com/google/syzkaller/pkg/report"
+	"github.com/google/syzkaller/syz-manager/mgrconfig"
+	"github.com/google/syzkaller/vm"
+)
+
+// bootInstance boots one VM using the provided config.
+// Returns either instance and reporter, or report with boot failure, or error.
+func bootInstance(mgrcfg *mgrconfig.Config) (*vm.Instance, report.Reporter, *report.Report, error) {
+	reporter, err := report.NewReporter(mgrcfg.TargetOS, mgrcfg.Kernel_Src,
+		filepath.Dir(mgrcfg.Vmlinux), nil, mgrcfg.ParsedIgnores)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	vmEnv := mgrconfig.CreateVMEnv(mgrcfg, false)
+	vmPool, err := vm.Create(mgrcfg.Type, vmEnv)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to create VM pool: %v", err)
+	}
+	inst, err := vmPool.Create(0)
+	if err != nil {
+		if bootErr, ok := err.(vm.BootError); ok {
+			rep := reporter.Parse(bootErr.Output)
+			if rep == nil {
+				rep = &report.Report{
+					Title:  bootErr.Title,
+					Output: bootErr.Output,
+				}
+			}
+			if err := reporter.Symbolize(rep); err != nil {
+				// TODO(dvyukov): send such errors to dashboard.
+				Logf(0, "failed to symbolize report: %v", err)
+			}
+			return nil, nil, rep, nil
+		}
+		return nil, nil, nil, fmt.Errorf("failed to create VM: %v", err)
+	}
+	return inst, reporter, nil, nil
+}
+
+// testInstance tests basic operation of the provided VM
+// (that we can copy binaries, run binaries, they can connect to host, run syzkaller programs, etc).
+// It either returns crash report if there is a kernel bug,
+// or err if there is an internal problem, or all nil's if testing succeeded.
+func testInstance(inst *vm.Instance, reporter report.Reporter, mgrcfg *mgrconfig.Config) (
+	*report.Report, error) {
+	ln, err := net.Listen("tcp", ":")
+	if err != nil {
+		return nil, fmt.Errorf("failed to open listening socket: %v", err)
+	}
+	defer ln.Close()
+	var gotConn uint32
+	go func() {
+		conn, err := ln.Accept()
+		if err == nil {
+			conn.Close()
+			atomic.StoreUint32(&gotConn, 1)
+		}
+		return
+	}()
+	fwdAddr, err := inst.Forward(ln.Addr().(*net.TCPAddr).Port)
+	if err != nil {
+		return nil, fmt.Errorf("failed to setup port forwarding: %v", err)
+	}
+	fuzzerBin, err := inst.Copy(mgrcfg.SyzFuzzerBin)
+	if err != nil {
+		return nil, fmt.Errorf("failed to copy test binary to VM: %v", err)
+	}
+	executorBin, err := inst.Copy(mgrcfg.SyzExecutorBin)
+	if err != nil {
+		return nil, fmt.Errorf("failed to copy test binary to VM: %v", err)
+	}
+	cmd := fmt.Sprintf("%v -test -executor=%v -name=test -arch=%v -manager=%v -cover=%v -sandbox=%v",
+		fuzzerBin, executorBin, mgrcfg.TargetArch, fwdAddr, mgrcfg.Cover, mgrcfg.Sandbox)
+	outc, errc, err := inst.Run(5*time.Minute, nil, cmd)
+	if err != nil {
+		return nil, fmt.Errorf("failed to run binary in VM: %v", err)
+	}
+	rep := vm.MonitorExecution(outc, errc, reporter, true)
+	if rep != nil {
+		if err := reporter.Symbolize(rep); err != nil {
+			// TODO(dvyukov): send such errors to dashboard.
+			Logf(0, "failed to symbolize report: %v", err)
+		}
+		return rep, nil
+	}
+	if atomic.LoadUint32(&gotConn) == 0 {
+		return nil, fmt.Errorf("test machine failed to connect to host")
+	}
+	return nil, nil
+}
diff --git a/syz-fuzzer/fuzzer.go b/syz-fuzzer/fuzzer.go
index ea7b1fcff..3b033b367 100644
--- a/syz-fuzzer/fuzzer.go
+++ b/syz-fuzzer/fuzzer.go
@@ -40,6 +40,7 @@ var (
 	flagLeak     = flag.Bool("leak", false, "detect memory leaks")
 	flagOutput   = flag.String("output", "stdout", "write programs to none/stdout/dmesg/file")
 	flagPprof    = flag.String("pprof", "", "address to serve pprof profiles")
+	flagTest     = flag.Bool("test", false, "enable image testing mode") // used by syz-ci
 )
 
 const (
@@ -121,6 +122,11 @@ func main() {
 		os.Exit(1)
 	}()
 
+	if *flagTest {
+		testImage(*flagManager, target)
+		return
+	}
+
 	if *flagPprof != "" {
 		go func() {
 			err := http.ListenAndServe(*flagPprof, nil)
@@ -223,10 +229,8 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	if _, ok := calls[target.SyscallMap["syz_emit_ethernet"]]; ok {
-		config.Flags |= ipc.FlagEnableTun
-	}
-	if _, ok := calls[target.SyscallMap["syz_extract_tcp_res"]]; ok {
+	if calls[target.SyscallMap["syz_emit_ethernet"]] ||
+		calls[target.SyscallMap["syz_extract_tcp_res"]] {
 		config.Flags |= ipc.FlagEnableTun
 	}
 	if faultInjectionEnabled {
diff --git a/syz-fuzzer/testing.go b/syz-fuzzer/testing.go
new file mode 100644
index 000000000..c3ebf9f85
--- /dev/null
+++ b/syz-fuzzer/testing.go
@@ -0,0 +1,72 @@
+// Copyright 2017 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"net"
+
+	"github.com/google/syzkaller/pkg/host"
+	"github.com/google/syzkaller/pkg/ipc"
+	. "github.com/google/syzkaller/pkg/log"
+	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/prog"
+)
+
+func testImage(hostAddr string, target *prog.Target) {
+	Logf(0, "connecting to host at %v", hostAddr)
+	conn, err := net.Dial("tcp", hostAddr)
+	if err != nil {
+		Fatalf("failed to connect: %v", err)
+	}
+	conn.Close()
+
+	Logf(0, "checking config...")
+	config, err := ipc.DefaultConfig()
+	if err != nil {
+		Fatalf("failed to create ipc config: %v", err)
+	}
+	if kcov, _ := checkCompsSupported(); !kcov && config.Flags&ipc.FlagSignal != 0 {
+		Fatalf("coverage is not supported by kernel")
+	}
+	if config.Flags&ipc.FlagSandboxNamespace != 0 && !osutil.IsExist("/proc/self/ns/user") {
+		Fatalf("/proc/self/ns/user is not present for namespace sandbox")
+	}
+	calls, err := host.DetectSupportedSyscalls(target)
+	if err != nil {
+		Fatalf("failed to detect supported syscalls: %v", err)
+	}
+	calls = target.TransitivelyEnabledCalls(calls)
+	Logf(0, "enabled syscalls: %v", len(calls))
+	if calls[target.SyscallMap["syz_emit_ethernet"]] ||
+		calls[target.SyscallMap["syz_extract_tcp_res"]] {
+		config.Flags |= ipc.FlagEnableTun
+	}
+
+	Logf(0, "testing simple program...")
+	env, err := ipc.MakeEnv(*flagExecutor, 0, config)
+	if err != nil {
+		Fatalf("failed to create ipc env: %v", err)
+	}
+	p := target.GenerateSimpleProg()
+	opts := &ipc.ExecOpts{}
+	output, info, failed, hanged, err := env.Exec(opts, p)
+	if err != nil {
+		Fatalf("execution failed: %v\n%s", err, output)
+	}
+	if hanged {
+		Fatalf("program hanged:\n%s", output)
+	}
+	if failed {
+		Fatalf("program failed:\n%s", output)
+	}
+	if len(info) == 0 {
+		Fatalf("no calls executed:\n%s", output)
+	}
+	if info[0].Errno != 0 {
+		Fatalf("simple call failed: %v\n%s", info[0].Errno, output)
+	}
+	if config.Flags&ipc.FlagSignal != 0 && len(info[0].Signal) == 0 {
+		Fatalf("got no coverage:\n%s", output)
+	}
+}
diff --git a/vm/gce/gce.go b/vm/gce/gce.go
index 96cc14e79..792982560 100644
--- a/vm/gce/gce.go
+++ b/vm/gce/gce.go
@@ -364,7 +364,7 @@ func (pool *Pool) waitInstanceBoot(name, ip, sshKey, sshUser, gceKey string) err
 	if err != nil {
 		output = []byte(fmt.Sprintf("failed to get boot output: %v", err))
 	}
-	return fmt.Errorf("can't ssh into the instance\n\n%s", output)
+	return vmimpl.BootError{"can't ssh into the instance", output}
 }
 
 func (pool *Pool) getSerialPortOutput(name, gceKey string) ([]byte, error) {
diff --git a/vm/qemu/qemu.go b/vm/qemu/qemu.go
index 679dcf25e..e3f33554a 100644
--- a/vm/qemu/qemu.go
+++ b/vm/qemu/qemu.go
@@ -373,13 +373,13 @@ func (inst *instance) Boot() error {
 			time.Sleep(time.Second) // wait for any pending output
 			bootOutputStop <- true
 			<-bootOutputStop
-			return fmt.Errorf("qemu stopped:\n%v\n", string(bootOutput))
+			return vmimpl.BootError{"qemu stopped", bootOutput}
 		default:
 		}
 		if time.Since(start) > 10*time.Minute {
 			bootOutputStop <- true
 			<-bootOutputStop
-			return fmt.Errorf("ssh server did not start:\n%v\n", string(bootOutput))
+			return vmimpl.BootError{"ssh server did not start", bootOutput}
 		}
 	}
 	bootOutputStop <- true
diff --git a/vm/vm.go b/vm/vm.go
index b59c0d223..48b68d597 100644
--- a/vm/vm.go
+++ b/vm/vm.go
@@ -37,13 +37,20 @@ type Instance struct {
 	index   int
 }
 
-type Env vmimpl.Env
+type (
+	Env       vmimpl.Env
+	BootError vmimpl.BootError
+)
 
 var (
 	Shutdown   = vmimpl.Shutdown
 	TimeoutErr = vmimpl.TimeoutErr
 )
 
+func (err BootError) Error() string {
+	return fmt.Sprintf("%v\n%s", err.Title, err.Output)
+}
+
 func Create(typ string, env *Env) (*Pool, error) {
 	impl, err := vmimpl.Create(typ, (*vmimpl.Env)(env))
 	if err != nil {
diff --git a/vm/vmimpl/vmimpl.go b/vm/vmimpl/vmimpl.go
index 617f9bc0f..81f798d26 100644
--- a/vm/vmimpl/vmimpl.go
+++ b/vm/vmimpl/vmimpl.go
@@ -56,6 +56,16 @@ type Env struct {
 	Config  []byte // json-serialized VM-type-specific config
 }
 
+// BootError is returned by Pool.Create when VM does not boot.
+type BootError struct {
+	Title  string
+	Output []byte
+}
+
+func (err BootError) Error() string {
+	return fmt.Sprintf("%v\n%s", err.Title, err.Output)
+}
+
 // Create creates a VM type that can be used to create individual VMs.
 func Create(typ string, env *Env) (Pool, error) {
 	ctor := ctors[typ]