pkg/instance: add package for testing of images/patches/bisection

Move helper image/patch testing code from syz-ci/testing.go
to a separate package so that it can be reused during bisection.

Update #501

4 files changed, 427 insertions, 313 deletions

diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go
new file mode 100644
index 000000000..163821c37
--- /dev/null
+++ b/pkg/instance/instance.go
@@ -0,0 +1,337 @@
+// Copyright 2018 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+// Package instance provides helper functions for creation of temporal instances
+// used for testing of images, patches and bisection.
+package instance
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/google/syzkaller/pkg/csource"
+	"github.com/google/syzkaller/pkg/git"
+	"github.com/google/syzkaller/pkg/kernel"
+	"github.com/google/syzkaller/pkg/log"
+	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/pkg/report"
+	"github.com/google/syzkaller/prog"
+	"github.com/google/syzkaller/syz-manager/mgrconfig"
+	"github.com/google/syzkaller/vm"
+)
+
+type Env struct {
+	cfg    *mgrconfig.Config
+	gopath string
+}
+
+func NewEnv(cfg *mgrconfig.Config) (*Env, error) {
+	switch cfg.Type {
+	case "gce", "qemu":
+	default:
+		return nil, fmt.Errorf("test instances can only work with qemu/gce")
+	}
+	if cfg.Workdir == "" {
+		return nil, fmt.Errorf("workdir path is empty")
+	}
+	if cfg.KernelSrc == "" {
+		return nil, fmt.Errorf("kernel src path is empty")
+	}
+	if cfg.Syzkaller == "" {
+		return nil, fmt.Errorf("syzkaller path is empty")
+	}
+	srcIndex := strings.LastIndex(cfg.Syzkaller, "/src/")
+	if srcIndex == -1 {
+		return nil, fmt.Errorf("syzkaller path %q is not in GOPATH", cfg.Syzkaller)
+	}
+	if err := osutil.MkdirAll(cfg.Workdir); err != nil {
+		return nil, fmt.Errorf("failed to create tmp dir: %v", err)
+	}
+	env := &Env{
+		cfg:    cfg,
+		gopath: cfg.Syzkaller[:srcIndex],
+	}
+	return env, nil
+}
+
+func (env *Env) BuildSyzkaller(repo, commit string) error {
+	if _, err := git.CheckoutCommit(env.cfg.Syzkaller, repo, commit); err != nil {
+		return fmt.Errorf("failed to checkout syzkaller repo: %v", err)
+	}
+	cmd := osutil.Command("make", "target")
+	cmd.Dir = env.cfg.Syzkaller
+	cmd.Env = append([]string{}, os.Environ()...)
+	cmd.Env = append(cmd.Env,
+		"GOPATH="+env.gopath,
+		"TARGETOS="+env.cfg.TargetOS,
+		"TARGETVMARCH="+env.cfg.TargetVMArch,
+		"TARGETARCH="+env.cfg.TargetArch,
+	)
+	if _, err := osutil.Run(time.Hour, cmd); err != nil {
+		return fmt.Errorf("syzkaller build failed: %v", err)
+	}
+	return nil
+}
+
+func (env *Env) BuildKernel(compilerBin, userspaceDir, cmdlineFile, sysctlFile string, kernelConfig []byte) error {
+	if err := kernel.Build(env.cfg.KernelSrc, compilerBin, kernelConfig); err != nil {
+		return fmt.Errorf("kernel build failed: %v", err)
+	}
+	env.cfg.Vmlinux = filepath.Join(env.cfg.KernelSrc, "vmlinux")
+	env.cfg.Image = filepath.Join(env.cfg.Workdir, "syz-image")
+	env.cfg.SSHKey = filepath.Join(env.cfg.Workdir, "syz-key")
+	if err := kernel.CreateImage(env.cfg.KernelSrc, userspaceDir,
+		cmdlineFile, sysctlFile, env.cfg.Image, env.cfg.SSHKey); err != nil {
+		return fmt.Errorf("image build failed: %v", err)
+	}
+	return nil
+}
+
+type TestError struct {
+	Boot   bool // says if the error happened during booting or during instance testing
+	Title  string
+	Output []byte
+	Report *report.Report
+}
+
+func (err *TestError) Error() string {
+	return err.Title
+}
+
+type CrashError struct {
+	Report *report.Report
+}
+
+func (err *CrashError) Error() string {
+	return err.Report.Title
+}
+
+// Test boots numVMs VMs, tests basic kernel operation, and optionally tests the provided reproducer.
+// TestError is returned if there is a problem with kernel/image (crash, reboot loop, etc).
+// CrashError is returned if the reproducer crashes kernel.
+func (env *Env) Test(numVMs int, reproSyz, reproOpts, reproC []byte) ([]error, error) {
+	if err := mgrconfig.Complete(env.cfg); err != nil {
+		return nil, err
+	}
+	reporter, err := report.NewReporter(env.cfg.TargetOS, env.cfg.KernelSrc,
+		filepath.Dir(env.cfg.Vmlinux), nil, env.cfg.ParsedIgnores)
+	if err != nil {
+		return nil, err
+	}
+	vmEnv := mgrconfig.CreateVMEnv(env.cfg, false)
+	vmPool, err := vm.Create(env.cfg.Type, vmEnv)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create VM pool: %v", err)
+	}
+	if n := vmPool.Count(); numVMs > n {
+		numVMs = n
+	}
+	res := make(chan error, numVMs)
+	for i := 0; i < numVMs; i++ {
+		inst := &inst{
+			cfg:       env.cfg,
+			reporter:  reporter,
+			vmPool:    vmPool,
+			vmIndex:   i,
+			reproSyz:  reproSyz,
+			reproOpts: reproOpts,
+			reproC:    reproC,
+		}
+		go func() { res <- inst.test() }()
+	}
+	var errors []error
+	for i := 0; i < numVMs; i++ {
+		errors = append(errors, <-res)
+	}
+	return errors, nil
+}
+
+type inst struct {
+	cfg       *mgrconfig.Config
+	reporter  report.Reporter
+	vmPool    *vm.Pool
+	vm        *vm.Instance
+	vmIndex   int
+	reproSyz  []byte
+	reproOpts []byte
+	reproC    []byte
+}
+
+func (inst *inst) test() error {
+	vmInst, err := inst.vmPool.Create(inst.vmIndex)
+	if err != nil {
+		testErr := &TestError{
+			Boot:  true,
+			Title: err.Error(),
+		}
+		if bootErr, ok := err.(vm.BootErrorer); ok {
+			testErr.Title, testErr.Output = bootErr.BootError()
+			// This linux-ism avoids detecting any crash during boot as "unexpected kernel reboot".
+			output := testErr.Output
+			if pos := bytes.Index(output, []byte("Booting the kernel.")); pos != -1 {
+				output = output[pos+1:]
+			}
+			testErr.Report = inst.reporter.Parse(output)
+			if testErr.Report != nil {
+				testErr.Title = testErr.Report.Title
+			} else {
+				testErr.Report = &report.Report{
+					Title:  testErr.Title,
+					Output: testErr.Output,
+				}
+			}
+			if err := inst.reporter.Symbolize(testErr.Report); err != nil {
+				// TODO(dvyukov): send such errors to dashboard.
+				log.Logf(0, "failed to symbolize report: %v", err)
+			}
+		}
+		return testErr
+	}
+	defer vmInst.Close()
+	inst.vm = vmInst
+	if err := inst.testInstance(); err != nil {
+		return err
+	}
+	if len(inst.reproSyz) != 0 {
+		if err := inst.testRepro(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// testInstance tests basic operation of the provided VM
+// (that we can copy binaries, run binaries, they can connect to host, run syzkaller programs, etc).
+// TestError is returned if there is a problem with the kernel (e.g. crash).
+func (inst *inst) testInstance() error {
+	ln, err := net.Listen("tcp", ":")
+	if err != nil {
+		return fmt.Errorf("failed to open listening socket: %v", err)
+	}
+	defer ln.Close()
+	var gotConn uint32
+	go func() {
+		conn, err := ln.Accept()
+		if err == nil {
+			conn.Close()
+			atomic.StoreUint32(&gotConn, 1)
+		}
+	}()
+	fwdAddr, err := inst.vm.Forward(ln.Addr().(*net.TCPAddr).Port)
+	if err != nil {
+		return fmt.Errorf("failed to setup port forwarding: %v", err)
+	}
+	fuzzerBin, err := inst.vm.Copy(inst.cfg.SyzFuzzerBin)
+	if err != nil {
+		return &TestError{Title: fmt.Sprintf("failed to copy test binary to VM: %v", err)}
+	}
+	executorBin, err := inst.vm.Copy(inst.cfg.SyzExecutorBin)
+	if err != nil {
+		return &TestError{Title: fmt.Sprintf("failed to copy test binary to VM: %v", err)}
+	}
+	cmd := fmt.Sprintf("%v -test -executor=%v -name=test -arch=%v -manager=%v -cover=0 -sandbox=%v",
+		fuzzerBin, executorBin, inst.cfg.TargetArch, fwdAddr, inst.cfg.Sandbox)
+	outc, errc, err := inst.vm.Run(5*time.Minute, nil, cmd)
+	if err != nil {
+		return fmt.Errorf("failed to run binary in VM: %v", err)
+	}
+	rep := vm.MonitorExecution(outc, errc, inst.reporter, true)
+	if rep != nil {
+		if err := inst.reporter.Symbolize(rep); err != nil {
+			// TODO(dvyukov): send such errors to dashboard.
+			log.Logf(0, "failed to symbolize report: %v", err)
+		}
+		return &TestError{
+			Title:  rep.Title,
+			Report: rep,
+		}
+	}
+	if atomic.LoadUint32(&gotConn) == 0 {
+		return fmt.Errorf("test machine failed to connect to host")
+	}
+	return nil
+}
+
+func (inst *inst) testRepro() error {
+	cfg := inst.cfg
+	execprogBin, err := inst.vm.Copy(cfg.SyzExecprogBin)
+	if err != nil {
+		return &TestError{Title: fmt.Sprintf("failed to copy test binary to VM: %v", err)}
+	}
+	executorBin, err := inst.vm.Copy(cfg.SyzExecutorBin)
+	if err != nil {
+		return &TestError{Title: fmt.Sprintf("failed to copy test binary to VM: %v", err)}
+	}
+	progFile := filepath.Join(cfg.Workdir, "repro.prog")
+	if err := osutil.WriteFile(progFile, inst.reproSyz); err != nil {
+		return fmt.Errorf("failed to write temp file: %v", err)
+	}
+	vmProgFile, err := inst.vm.Copy(progFile)
+	if err != nil {
+		return &TestError{Title: fmt.Sprintf("failed to copy test binary to VM: %v", err)}
+	}
+	opts, err := csource.DeserializeOptions(inst.reproOpts)
+	if err != nil {
+		return err
+	}
+	// Combine repro options and default options in a way that increases chances to reproduce the crash.
+	// First, we always enable threaded/collide as it should be [almost] strictly better.
+	// Executor does not support empty sandbox, so we use none instead.
+	// Finally, always use repeat and multiple procs.
+	if opts.Sandbox == "" {
+		opts.Sandbox = "none"
+	}
+	if !opts.Fault {
+		opts.FaultCall = -1
+	}
+	cmdSyz := fmt.Sprintf("%v -executor %v -arch=%v -procs=%v -sandbox=%v"+
+		" -fault_call=%v -fault_nth=%v -repeat=0 -cover=0 %v",
+		execprogBin, executorBin, cfg.TargetArch, cfg.Procs, opts.Sandbox,
+		opts.FaultCall, opts.FaultNth, vmProgFile)
+	if err := inst.testProgram(cmdSyz, 7*time.Minute); err != nil {
+		return err
+	}
+	if len(inst.reproC) == 0 {
+		return nil
+	}
+	cFile := filepath.Join(cfg.Workdir, "repro.c")
+	if err := osutil.WriteFile(cFile, inst.reproC); err != nil {
+		return fmt.Errorf("failed to write temp file: %v", err)
+	}
+	target, err := prog.GetTarget(cfg.TargetOS, cfg.TargetArch)
+	if err != nil {
+		return err
+	}
+	bin, err := csource.Build(target, "c", cFile)
+	if err != nil {
+		return err
+	}
+	vmBin, err := inst.vm.Copy(bin)
+	if err != nil {
+		return &TestError{Title: fmt.Sprintf("failed to copy test binary to VM: %v", err)}
+	}
+	// We should test for longer (e.g. 5 mins), but the problem is that
+	// reproducer does not print anything, so after 3 mins we detect "no output".
+	return inst.testProgram(vmBin, time.Minute)
+}
+
+func (inst *inst) testProgram(command string, testTime time.Duration) error {
+	outc, errc, err := inst.vm.Run(testTime, nil, command)
+	if err != nil {
+		return fmt.Errorf("failed to run binary in VM: %v", err)
+	}
+	rep := vm.MonitorExecution(outc, errc, inst.reporter, true)
+	if rep == nil {
+		return nil
+	}
+	if err := inst.reporter.Symbolize(rep); err != nil {
+		log.Logf(0, "failed to symbolize report: %v", err)
+	}
+	return &CrashError{Report: rep}
+}
diff --git a/syz-ci/jobs.go b/syz-ci/jobs.go
index bda2df703..d4b1e51b5 100644
--- a/syz-ci/jobs.go
+++ b/syz-ci/jobs.go
@@ -11,15 +11,12 @@ import (
 	"time"
 
 	"github.com/google/syzkaller/dashboard/dashapi"
-	"github.com/google/syzkaller/pkg/csource"
 	"github.com/google/syzkaller/pkg/git"
+	"github.com/google/syzkaller/pkg/instance"
 	"github.com/google/syzkaller/pkg/kernel"
 	"github.com/google/syzkaller/pkg/log"
 	"github.com/google/syzkaller/pkg/osutil"
-	"github.com/google/syzkaller/pkg/report"
-	"github.com/google/syzkaller/prog"
 	"github.com/google/syzkaller/syz-manager/mgrconfig"
-	"github.com/google/syzkaller/vm"
 )
 
 type JobProcessor struct {
@@ -100,10 +97,9 @@ func (jp *JobProcessor) poll() {
 }
 
 type Job struct {
-	req    *dashapi.JobPollResp
-	resp   *dashapi.JobDoneReq
-	mgr    *Manager
-	mgrcfg *mgrconfig.Config
+	req  *dashapi.JobPollResp
+	resp *dashapi.JobDoneReq
+	mgr  *Manager
 }
 
 func (jp *JobProcessor) process(job *Job) *dashapi.JobDoneReq {
@@ -153,69 +149,40 @@ func (jp *JobProcessor) process(job *Job) *dashapi.JobDoneReq {
 		jp.Errorf("%s", job.resp.Error)
 		return job.resp
 	}
-	if err := jp.buildImage(job); err != nil {
-		job.resp.Error = []byte(err.Error())
-		return job.resp
-	}
-	var err error
-	for try := 0; try < 3; try++ {
-		if err = jp.test(job); err == nil {
-			break
-		}
-		log.Logf(0, "job: testing failed, trying once again\n%v", err)
-	}
-	if err != nil {
+	if err := jp.test(job); err != nil {
 		job.resp.Error = []byte(err.Error())
 	}
 	return job.resp
 }
 
-func (jp *JobProcessor) buildImage(job *Job) error {
+func (jp *JobProcessor) test(job *Job) error {
 	kernelBuildSem <- struct{}{}
 	defer func() { <-kernelBuildSem }()
 	req, resp, mgr := job.req, job.resp, job.mgr
 
 	dir := osutil.Abs(filepath.Join("jobs", mgr.managercfg.TargetOS))
 	kernelDir := filepath.Join(dir, "kernel")
-	if err := osutil.MkdirAll(kernelDir); err != nil {
-		return fmt.Errorf("failed to create temp dir: %v", err)
-	}
-	imageDir := filepath.Join(dir, "image")
-	os.RemoveAll(imageDir)
-	if err := osutil.MkdirAll(imageDir); err != nil {
-		return fmt.Errorf("failed to create temp dir: %v", err)
-	}
-	workDir := filepath.Join(dir, "workdir")
-	os.RemoveAll(workDir)
-	if err := osutil.MkdirAll(workDir); err != nil {
-		return fmt.Errorf("failed to create temp dir: %v", err)
-	}
-	gopathDir := filepath.Join(dir, "gopath")
-	syzkallerDir := filepath.Join(gopathDir, "src", "github.com", "google", "syzkaller")
-	if err := osutil.MkdirAll(syzkallerDir); err != nil {
-		return fmt.Errorf("failed to create temp dir: %v", err)
-	}
 
-	log.Logf(0, "job: fetching syzkaller on %v...", req.SyzkallerCommit)
-	_, err := git.CheckoutCommit(syzkallerDir, jp.syzkallerRepo, req.SyzkallerCommit)
-	if err != nil {
-		return fmt.Errorf("failed to checkout syzkaller repo: %v", err)
-	}
+	mgrcfg := new(mgrconfig.Config)
+	*mgrcfg = *mgr.managercfg
+	mgrcfg.Name += "-job"
+	mgrcfg.Workdir = filepath.Join(dir, "workdir")
+	mgrcfg.KernelSrc = kernelDir
+	mgrcfg.Vmlinux = filepath.Join(kernelDir, "vmlinux")
+	mgrcfg.Syzkaller = filepath.Join(dir, "gopath", "src", "github.com", "google", "syzkaller")
 
-	log.Logf(0, "job: building syzkaller...")
-	cmd := osutil.Command("make", "target")
-	cmd.Dir = syzkallerDir
-	cmd.Env = append([]string{}, os.Environ()...)
-	cmd.Env = append(cmd.Env,
-		"GOPATH="+gopathDir,
-		"TARGETOS="+mgr.managercfg.TargetOS,
-		"TARGETVMARCH="+mgr.managercfg.TargetVMArch,
-		"TARGETARCH="+mgr.managercfg.TargetArch,
-	)
-	if _, err := osutil.Run(time.Hour, cmd); err != nil {
-		return fmt.Errorf("syzkaller build failed: %v", err)
+	os.RemoveAll(mgrcfg.Workdir)
+	defer os.RemoveAll(mgrcfg.Workdir)
+
+	env, err := instance.NewEnv(mgrcfg)
+	if err != nil {
+		return err
 	}
+	log.Logf(0, "job: building syzkaller on %v...", req.SyzkallerCommit)
 	resp.Build.SyzkallerCommit = req.SyzkallerCommit
+	if err := env.BuildSyzkaller(jp.syzkallerRepo, req.SyzkallerCommit); err != nil {
+		return err
+	}
 
 	log.Logf(0, "job: fetching kernel...")
 	var kernelCommit *git.Commit
@@ -247,152 +214,46 @@ func (jp *JobProcessor) buildImage(job *Job) error {
 	}
 
 	log.Logf(0, "job: building kernel...")
-	if err := kernel.Build(kernelDir, mgr.mgrcfg.Compiler, req.KernelConfig); err != nil {
-		return fmt.Errorf("kernel build failed: %v", err)
-	}
-	resp.Build.KernelConfig, err = ioutil.ReadFile(filepath.Join(kernelDir, ".config"))
-	if err != nil {
-		return fmt.Errorf("failed to read config file: %v", err)
-	}
-
-	log.Logf(0, "job: creating image...")
-	image := filepath.Join(imageDir, "image")
-	key := filepath.Join(imageDir, "key")
-	err = kernel.CreateImage(kernelDir, mgr.mgrcfg.Userspace,
-		mgr.mgrcfg.KernelCmdline, mgr.mgrcfg.KernelSysctl, image, key)
-	if err != nil {
-		return fmt.Errorf("image build failed: %v", err)
-	}
-
-	mgrcfg := new(mgrconfig.Config)
-	*mgrcfg = *mgr.managercfg
-	mgrcfg.Name += "-job"
-	mgrcfg.Workdir = workDir
-	mgrcfg.Vmlinux = filepath.Join(kernelDir, "vmlinux")
-	mgrcfg.KernelSrc = kernelDir
-	mgrcfg.Syzkaller = syzkallerDir
-	mgrcfg.Image = image
-	mgrcfg.SSHKey = key
-	if err := mgrconfig.Complete(mgrcfg); err != nil {
-		return fmt.Errorf("bad manager config: %v", err)
-	}
-	job.mgrcfg = mgrcfg
-	return nil
-}
-
-func (jp *JobProcessor) test(job *Job) error {
-	req, mgrcfg := job.req, job.mgrcfg
-
-	log.Logf(0, "job: booting VM...")
-	inst, reporter, rep, err := bootInstance(mgrcfg)
-	if err != nil {
+	if err := env.BuildKernel(mgr.mgrcfg.Compiler, mgr.mgrcfg.Userspace, mgr.mgrcfg.KernelCmdline,
+		mgr.mgrcfg.KernelSysctl, req.KernelConfig); err != nil {
 		return err
 	}
-	if rep != nil {
-		// We should not put rep into resp.CrashTitle/CrashReport,
-		// because that will be treated as patch not fixing the bug.
-		return fmt.Errorf("%v\n\n%s\n\n%s", rep.Title, rep.Report, rep.Output)
-	}
-	defer inst.Close()
-
-	log.Logf(0, "job: testing instance...")
-	rep, err = testInstance(inst, reporter, mgrcfg)
+	resp.Build.KernelConfig, err = ioutil.ReadFile(filepath.Join(mgrcfg.KernelSrc, ".config"))
 	if err != nil {
-		return err
-	}
-	if rep != nil {
-		// We should not put rep into resp.CrashTitle/CrashReport,
-		// because that will be treated as patch not fixing the bug.
-		return fmt.Errorf("%v\n\n%s\n\n%s", rep.Title, rep.Report, rep.Output)
-	}
-
-	log.Logf(0, "job: copying binaries...")
-	execprogBin, err := inst.Copy(mgrcfg.SyzExecprogBin)
-	if err != nil {
-		return fmt.Errorf("failed to copy test binary to VM: %v", err)
-	}
-	executorBin, err := inst.Copy(mgrcfg.SyzExecutorBin)
-	if err != nil {
-		return fmt.Errorf("failed to copy test binary to VM: %v", err)
-	}
-	progFile := filepath.Join(mgrcfg.Workdir, "repro.prog")
-	if err := osutil.WriteFile(progFile, req.ReproSyz); err != nil {
-		return fmt.Errorf("failed to write temp file: %v", err)
-	}
-	vmProgFile, err := inst.Copy(progFile)
-	if err != nil {
-		return fmt.Errorf("failed to copy to VM: %v", err)
+		return fmt.Errorf("failed to read config file: %v", err)
 	}
 
-	log.Logf(0, "job: testing syzkaller program...")
-	opts, err := csource.DeserializeOptions(req.ReproOpts)
+	log.Logf(0, "job: testing...")
+	results, err := env.Test(3, req.ReproSyz, req.ReproOpts, req.ReproC)
 	if err != nil {
 		return err
 	}
-	// Combine repro options and default options in a way that increases chances to reproduce the crash.
-	// First, we always enable threaded/collide as it should be [almost] strictly better.
-	// Executor does not support empty sandbox, so we use none instead.
-	// Finally, always use repeat and multiple procs.
-	if opts.Sandbox == "" {
-		opts.Sandbox = "none"
-	}
-	if !opts.Fault {
-		opts.FaultCall = -1
-	}
-	cmdSyz := fmt.Sprintf("%v -executor %v -arch=%v -procs=%v -sandbox=%v"+
-		" -fault_call=%v -fault_nth=%v -repeat=0 -cover=0 %v",
-		execprogBin, executorBin, mgrcfg.TargetArch, mgrcfg.Procs, opts.Sandbox,
-		opts.FaultCall, opts.FaultNth, vmProgFile)
-	crashed, err := jp.testProgram(job, inst, cmdSyz, reporter, 7*time.Minute)
-	if crashed || err != nil {
-		return err
-	}
-
-	if len(req.ReproC) != 0 {
-		log.Logf(0, "job: testing C program...")
-		cFile := filepath.Join(mgrcfg.Workdir, "repro.c")
-		if err := osutil.WriteFile(cFile, req.ReproC); err != nil {
-			return fmt.Errorf("failed to write temp file: %v", err)
-		}
-		target, err := prog.GetTarget(mgrcfg.TargetOS, mgrcfg.TargetArch)
-		if err != nil {
-			return err
-		}
-		bin, err := csource.Build(target, "c", cFile)
-		if err != nil {
-			return err
+	var anyErr, testErr error
+	for _, res := range results {
+		if res == nil {
+			continue
 		}
-		vmBin, err := inst.Copy(bin)
-		if err != nil {
-			return fmt.Errorf("failed to copy test binary to VM: %v", err)
+		anyErr = res
+		switch err := res.(type) {
+		case *instance.TestError:
+			// We should not put rep into resp.CrashTitle/CrashReport,
+			// because that will be treated as patch not fixing the bug.
+			if rep := err.Report; rep != nil {
+				testErr = fmt.Errorf("%v\n\n%s\n\n%s", rep.Title, rep.Report, rep.Output)
+			} else {
+				testErr = fmt.Errorf("%v\n\n%s", err.Title, err.Output)
+			}
+		case *instance.CrashError:
+			resp.CrashTitle = err.Report.Title
+			resp.CrashReport = err.Report.Report
+			resp.CrashLog = err.Report.Output
+			return nil
 		}
-		// We should test for longer (e.g. 5 mins), but the problem is that
-		// reproducer does not print anything, so after 3 mins we detect "no output".
-		crashed, err := jp.testProgram(job, inst, vmBin, reporter, time.Minute)
-		if crashed || err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (jp *JobProcessor) testProgram(job *Job, inst *vm.Instance, command string,
-	reporter report.Reporter, testTime time.Duration) (bool, error) {
-	outc, errc, err := inst.Run(testTime, nil, command)
-	if err != nil {
-		return false, fmt.Errorf("failed to run binary in VM: %v", err)
-	}
-	rep := vm.MonitorExecution(outc, errc, reporter, true)
-	if rep == nil {
-		return false, nil
 	}
-	if err := reporter.Symbolize(rep); err != nil {
-		jp.Errorf("failed to symbolize report: %v", err)
+	if testErr != nil {
+		return testErr
 	}
-	job.resp.CrashTitle = rep.Title
-	job.resp.CrashReport = rep.Report
-	job.resp.CrashLog = rep.Output
-	return true, nil
+	return anyErr
 }
 
 // Errorf logs non-fatal error and sends it to dashboard.
diff --git a/syz-ci/manager.go b/syz-ci/manager.go
index 00ef66af8..dfc0f92af 100644
--- a/syz-ci/manager.go
+++ b/syz-ci/manager.go
@@ -14,6 +14,7 @@ import (
 	"github.com/google/syzkaller/pkg/config"
 	"github.com/google/syzkaller/pkg/git"
 	"github.com/google/syzkaller/pkg/hash"
+	"github.com/google/syzkaller/pkg/instance"
 	"github.com/google/syzkaller/pkg/kernel"
 	"github.com/google/syzkaller/pkg/log"
 	"github.com/google/syzkaller/pkg/osutil"
@@ -349,39 +350,57 @@ func (mgr *Manager) testImage(imageDir string, info *BuildInfo) error {
 	if err != nil {
 		return fmt.Errorf("failed to create manager config: %v", err)
 	}
+	defer os.RemoveAll(mgrcfg.Workdir)
 	switch typ := mgrcfg.Type; typ {
 	case "gce", "qemu":
 	default:
 		// Other types don't support creating machines out of thin air.
 		return nil
 	}
-	if err := osutil.MkdirAll(mgrcfg.Workdir); err != nil {
-		return fmt.Errorf("failed to create tmp dir: %v", err)
-	}
-	defer os.RemoveAll(mgrcfg.Workdir)
-
-	inst, reporter, rep, err := bootInstance(mgrcfg)
+	env, err := instance.NewEnv(mgrcfg)
 	if err != nil {
 		return err
 	}
-	if rep != nil {
-		rep.Title = fmt.Sprintf("%v boot error: %v", mgr.mgrcfg.RepoAlias, rep.Title)
-		if err := mgr.reportBuildError(rep, info, imageDir); err != nil {
-			mgr.Errorf("failed to report image error: %v", err)
-		}
-		return fmt.Errorf("VM boot failed with: %v", rep.Title)
-	}
-	defer inst.Close()
-	rep, err = testInstance(inst, reporter, mgrcfg)
+	const (
+		testVMs     = 3
+		maxFailures = 1
+	)
+	results, err := env.Test(testVMs, nil, nil, nil)
 	if err != nil {
 		return err
 	}
-	if rep != nil {
-		rep.Title = fmt.Sprintf("%v test error: %v", mgr.mgrcfg.RepoAlias, rep.Title)
-		if err := mgr.reportBuildError(rep, info, imageDir); err != nil {
-			mgr.Errorf("failed to report image error: %v", err)
+	failures := 0
+	var failureErr error
+	for _, res := range results {
+		if res == nil {
+			continue
 		}
-		return fmt.Errorf("VM testing failed with: %v", rep.Title)
+		failures++
+		switch err := res.(type) {
+		case *instance.TestError:
+			if rep := err.Report; rep != nil {
+				if err.Boot {
+					rep.Title = fmt.Sprintf("%v boot error: %v",
+						mgr.mgrcfg.RepoAlias, rep.Title)
+				} else {
+					rep.Title = fmt.Sprintf("%v test error: %v",
+						mgr.mgrcfg.RepoAlias, rep.Title)
+				}
+				if err := mgr.reportBuildError(rep, info, imageDir); err != nil {
+					mgr.Errorf("failed to report image error: %v", err)
+				}
+			}
+			if err.Boot {
+				failureErr = fmt.Errorf("VM boot failed with: %v", err)
+			} else {
+				failureErr = fmt.Errorf("VM testing failed with: %v", err)
+			}
+		default:
+			failureErr = res
+		}
+	}
+	if failures > maxFailures {
+		return failureErr
 	}
 	return nil
 }
diff --git a/syz-ci/testing.go b/syz-ci/testing.go
deleted file mode 100644
index 387be26e4..000000000
--- a/syz-ci/testing.go
+++ /dev/null
@@ -1,103 +0,0 @@
-// Copyright 2017 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package main
-
-import (
-	"fmt"
-	"net"
-	"path/filepath"
-	"sync/atomic"
-	"time"
-
-	"github.com/google/syzkaller/pkg/log"
-	"github.com/google/syzkaller/pkg/report"
-	"github.com/google/syzkaller/syz-manager/mgrconfig"
-	"github.com/google/syzkaller/vm"
-)
-
-// bootInstance boots one VM using the provided config.
-// Returns either instance and reporter, or report with boot failure, or error.
-func bootInstance(mgrcfg *mgrconfig.Config) (*vm.Instance, report.Reporter, *report.Report, error) {
-	reporter, err := report.NewReporter(mgrcfg.TargetOS, mgrcfg.KernelSrc,
-		filepath.Dir(mgrcfg.Vmlinux), nil, mgrcfg.ParsedIgnores)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-	vmEnv := mgrconfig.CreateVMEnv(mgrcfg, false)
-	vmPool, err := vm.Create(mgrcfg.Type, vmEnv)
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to create VM pool: %v", err)
-	}
-	inst, err := vmPool.Create(0)
-	if err != nil {
-		if bootErr, ok := err.(vm.BootErrorer); ok {
-			title, output := bootErr.BootError()
-			rep := reporter.Parse(output)
-			if rep == nil {
-				rep = &report.Report{
-					Title:  title,
-					Output: output,
-				}
-			}
-			if err := reporter.Symbolize(rep); err != nil {
-				// TODO(dvyukov): send such errors to dashboard.
-				log.Logf(0, "failed to symbolize report: %v", err)
-			}
-			return nil, nil, rep, nil
-		}
-		return nil, nil, nil, fmt.Errorf("failed to create VM: %v", err)
-	}
-	return inst, reporter, nil, nil
-}
-
-// testInstance tests basic operation of the provided VM
-// (that we can copy binaries, run binaries, they can connect to host, run syzkaller programs, etc).
-// It either returns crash report if there is a kernel bug,
-// or err if there is an internal problem, or all nil's if testing succeeded.
-func testInstance(inst *vm.Instance, reporter report.Reporter, mgrcfg *mgrconfig.Config) (
-	*report.Report, error) {
-	ln, err := net.Listen("tcp", ":")
-	if err != nil {
-		return nil, fmt.Errorf("failed to open listening socket: %v", err)
-	}
-	defer ln.Close()
-	var gotConn uint32
-	go func() {
-		conn, err := ln.Accept()
-		if err == nil {
-			conn.Close()
-			atomic.StoreUint32(&gotConn, 1)
-		}
-	}()
-	fwdAddr, err := inst.Forward(ln.Addr().(*net.TCPAddr).Port)
-	if err != nil {
-		return nil, fmt.Errorf("failed to setup port forwarding: %v", err)
-	}
-	fuzzerBin, err := inst.Copy(mgrcfg.SyzFuzzerBin)
-	if err != nil {
-		return nil, fmt.Errorf("failed to copy test binary to VM: %v", err)
-	}
-	executorBin, err := inst.Copy(mgrcfg.SyzExecutorBin)
-	if err != nil {
-		return nil, fmt.Errorf("failed to copy test binary to VM: %v", err)
-	}
-	cmd := fmt.Sprintf("%v -test -executor=%v -name=test -arch=%v -manager=%v -cover=%v -sandbox=%v",
-		fuzzerBin, executorBin, mgrcfg.TargetArch, fwdAddr, mgrcfg.Cover, mgrcfg.Sandbox)
-	outc, errc, err := inst.Run(5*time.Minute, nil, cmd)
-	if err != nil {
-		return nil, fmt.Errorf("failed to run binary in VM: %v", err)
-	}
-	rep := vm.MonitorExecution(outc, errc, reporter, true)
-	if rep != nil {
-		if err := reporter.Symbolize(rep); err != nil {
-			// TODO(dvyukov): send such errors to dashboard.
-			log.Logf(0, "failed to symbolize report: %v", err)
-		}
-		return rep, nil
-	}
-	if atomic.LoadUint32(&gotConn) == 0 {
-		return nil, fmt.Errorf("test machine failed to connect to host")
-	}
-	return nil, nil
-}