tools/syz-runtest: switch to the generic program execution

syz-runtest effectively implemented the same execute program/return result
mechanism we use now for normal fuzzing.
So extend the general mechanism to allow collecting output/errors,
repeating program, and executing a precompiled binary as a test.
And switch syz-runtest to the general mechanism.
This removes another chunk of code from syz-fuzzer.

11 files changed, 336 insertions, 311 deletions

diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go
index 7f1ca4dbb..3e59209e2 100644
--- a/pkg/instance/instance.go
+++ b/pkg/instance/instance.go
@@ -459,7 +459,6 @@ type OptionalFuzzerArgs struct {
 	Slowdown       int
 	SandboxArg     int
 	PprofPort      int
-	ResetAccState  bool
 	NetCompression bool
 }
 
@@ -476,7 +475,6 @@ type FuzzerCmdArgs struct {
 	Cover     bool
 	Debug     bool
 	Test      bool
-	Runtest   bool
 	Optional  *OptionalFuzzerArgs
 }
 
@@ -488,10 +486,6 @@ func FuzzerCmd(args *FuzzerCmdArgs) string {
 		// because old execprog does not have os flag.
 		osArg = " -os=" + args.OS
 	}
-	runtestArg := ""
-	if args.Runtest {
-		runtestArg = " -runtest"
-	}
 	verbosityArg := ""
 	if args.Verbosity != 0 {
 		verbosityArg = fmt.Sprintf(" -vv=%v", args.Verbosity)
@@ -502,15 +496,14 @@ func FuzzerCmd(args *FuzzerCmdArgs) string {
 			{Name: "slowdown", Value: fmt.Sprint(args.Optional.Slowdown)},
 			{Name: "sandbox_arg", Value: fmt.Sprint(args.Optional.SandboxArg)},
 			{Name: "pprof_port", Value: fmt.Sprint(args.Optional.PprofPort)},
-			{Name: "reset_acc_state", Value: fmt.Sprint(args.Optional.ResetAccState)},
 			{Name: "net_compression", Value: fmt.Sprint(args.Optional.NetCompression)},
 		}
 		optionalArg = " " + tool.OptionalFlags(flags)
 	}
 	return fmt.Sprintf("%v -executor=%v -name=%v -arch=%v%v -manager=%v -sandbox=%v"+
-		" -procs=%v -cover=%v -debug=%v -test=%v%v%v%v",
+		" -procs=%v -cover=%v -debug=%v -test=%v%v%v",
 		args.Fuzzer, args.Executor, args.Name, args.Arch, osArg, args.FwdAddr, args.Sandbox,
-		args.Procs, args.Cover, args.Debug, args.Test, runtestArg, verbosityArg, optionalArg)
+		args.Procs, args.Cover, args.Debug, args.Test, verbosityArg, optionalArg)
 }
 
 func OldFuzzerCmd(fuzzer, executor, name, OS, arch, fwdAddr, sandbox string, sandboxArg, procs int,
@@ -521,7 +514,7 @@ func OldFuzzerCmd(fuzzer, executor, name, OS, arch, fwdAddr, sandbox string, san
 	}
 	return FuzzerCmd(&FuzzerCmdArgs{Fuzzer: fuzzer, Executor: executor, Name: name,
 		OS: OS, Arch: arch, FwdAddr: fwdAddr, Sandbox: sandbox,
-		Procs: procs, Verbosity: 0, Cover: cover, Debug: false, Test: test, Runtest: false,
+		Procs: procs, Verbosity: 0, Cover: cover, Debug: false, Test: test,
 		Optional: optional})
 }
 
diff --git a/pkg/rpctype/rpctype.go b/pkg/rpctype/rpctype.go
index 64caf6a04..0c26c65e8 100644
--- a/pkg/rpctype/rpctype.go
+++ b/pkg/rpctype/rpctype.go
@@ -17,12 +17,23 @@ import (
 // ExecutionRequest describes the task of executing a particular program.
 // Corresponds to Fuzzer.Request.
 type ExecutionRequest struct {
-	ID               int64
-	ProgData         []byte
-	ExecOpts         ipc.ExecOpts
-	NewSignal        bool
+	ID        int64
+	ProgData  []byte
+	ExecOpts  ipc.ExecOpts
+	NewSignal bool
+	// If set, ProgData contains compiled executable binary
+	// that needs to be written to disk and executed.
+	IsBinary bool
+	// If set, fully reset executor state befor executing the test.
+	ResetState bool
+	// If set, collect program output and return in ExecutionResult.Output.
+	ReturnOutput bool
+	// If set, don't fail on program failures, instead return the error in ExecutionResult.Error.
+	ReturnError      bool
 	SignalFilter     signal.Signal
 	SignalFilterCall int
+	// Repeat the program that many times (0 means 1).
+	Repeat int
 }
 
 // ExecutionResult is sent after ExecutionRequest is completed.
@@ -31,6 +42,8 @@ type ExecutionResult struct {
 	ProcID int
 	Try    int
 	Info   ipc.ProgInfo
+	Output []byte
+	Error  string
 }
 
 // ExchangeInfoRequest is periodically sent by syz-fuzzer to syz-manager.
@@ -198,24 +211,3 @@ type HubInput struct {
 	Domain string
 	Prog   []byte
 }
-
-type RunTestPollReq struct {
-	Name string
-}
-
-type RunTestPollRes struct {
-	ID     int
-	Bin    []byte
-	Prog   []byte
-	Cfg    *ipc.Config
-	Opts   *ipc.ExecOpts
-	Repeat int
-}
-
-type RunTestDoneArgs struct {
-	Name   string
-	ID     int
-	Output []byte
-	Info   []*ipc.ProgInfo
-	Error  string
-}
diff --git a/pkg/runtest/run.go b/pkg/runtest/run.go
index 5c00bf982..796fb6040 100644
--- a/pkg/runtest/run.go
+++ b/pkg/runtest/run.go
@@ -13,7 +13,6 @@ package runtest
 import (
 	"bufio"
 	"bytes"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -22,12 +21,10 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/google/syzkaller/pkg/csource"
 	"github.com/google/syzkaller/pkg/host"
 	"github.com/google/syzkaller/pkg/ipc"
-	"github.com/google/syzkaller/pkg/osutil"
 	"github.com/google/syzkaller/prog"
 	"github.com/google/syzkaller/sys/targets"
 )
@@ -35,19 +32,19 @@ import (
 type RunRequest struct {
 	Bin    string
 	P      *prog.Prog
-	Cfg    *ipc.Config
-	Opts   *ipc.ExecOpts
+	Opts   ipc.ExecOpts
 	Repeat int
 
 	Done   chan struct{}
 	Output []byte
-	Info   []*ipc.ProgInfo
+	Info   ipc.ProgInfo
 	Err    error
 
-	results *ipc.ProgInfo
-	name    string
-	broken  string
-	skip    string
+	finished chan struct{}
+	results  *ipc.ProgInfo
+	name     string
+	broken   string
+	skip     string
 }
 
 type Context struct {
@@ -78,19 +75,15 @@ func (ctx *Context) Run() error {
 		defer close(progs)
 		errc <- ctx.generatePrograms(progs)
 	}()
-	var ok, fail, broken, skip int
+	var requests []*RunRequest
 	for req := range progs {
-		result := ""
-		verbose := false
-		if req.broken != "" {
-			broken++
-			result = fmt.Sprintf("BROKEN (%v)", req.broken)
-			verbose = true
-		} else if req.skip != "" {
-			skip++
-			result = fmt.Sprintf("SKIP (%v)", req.skip)
-			verbose = true
-		} else {
+		req := req
+		requests = append(requests, req)
+		if req.broken != "" || req.skip != "" {
+			continue
+		}
+		req.finished = make(chan struct{})
+		go func() {
 			// The tests depend on timings and may be flaky, esp on overloaded/slow machines.
 			// We don't want to fix this by significantly bumping all timeouts,
 			// because if a program fails all the time with the default timeouts,
@@ -102,7 +95,7 @@ func (ctx *Context) Run() error {
 			var resultErr error
 			for try, failed := 0, 0; try < ctx.Retries; try++ {
 				req.Output = nil
-				req.Info = nil
+				req.Info = ipc.ProgInfo{}
 				req.Done = make(chan struct{})
 				ctx.Requests <- req
 				<-req.Done
@@ -122,6 +115,23 @@ func (ctx *Context) Run() error {
 			if req.Err == nil {
 				req.Err = resultErr
 			}
+			close(req.finished)
+		}()
+	}
+	var ok, fail, broken, skip int
+	for _, req := range requests {
+		result := ""
+		verbose := false
+		if req.broken != "" {
+			broken++
+			result = fmt.Sprintf("BROKEN (%v)", req.broken)
+			verbose = true
+		} else if req.skip != "" {
+			skip++
+			result = fmt.Sprintf("SKIP (%v)", req.skip)
+			verbose = true
+		} else {
+			<-req.finished
 			if req.Err != nil {
 				fail++
 				result = fmt.Sprintf("FAIL: %v",
@@ -400,12 +410,7 @@ func match(props, requires map[string]bool) bool {
 }
 
 func (ctx *Context) createSyzTest(p *prog.Prog, sandbox string, threaded, cov bool, times int) (*RunRequest, error) {
-	sysTarget := targets.Get(p.Target.OS, p.Target.Arch)
-	cfg := new(ipc.Config)
-	opts := new(ipc.ExecOpts)
-	cfg.UseShmem = sysTarget.ExecutorUsesShmem
-	cfg.UseForkServer = sysTarget.ExecutorUsesForkServer
-	cfg.Timeouts = sysTarget.Timeouts(1)
+	var opts ipc.ExecOpts
 	sandboxFlags, err := ipc.SandboxToFlags(sandbox)
 	if err != nil {
 		return nil, err
@@ -425,7 +430,6 @@ func (ctx *Context) createSyzTest(p *prog.Prog, sandbox string, threaded, cov bo
 	}
 	req := &RunRequest{
 		P:      p,
-		Cfg:    cfg,
 		Opts:   opts,
 		Repeat: times,
 	}
@@ -478,7 +482,7 @@ func (ctx *Context) createCTest(p *prog.Prog, sandbox string, threaded bool, tim
 	req := &RunRequest{
 		P:   p,
 		Bin: bin,
-		Opts: &ipc.ExecOpts{
+		Opts: ipc.ExecOpts{
 			ExecFlags: ipcFlags,
 		},
 		Repeat: times,
@@ -487,19 +491,30 @@ func (ctx *Context) createCTest(p *prog.Prog, sandbox string, threaded bool, tim
 }
 
 func checkResult(req *RunRequest) error {
+	var infos []ipc.ProgInfo
 	isC := req.Bin != ""
 	if isC {
 		var err error
-		if req.Info, err = parseBinOutput(req); err != nil {
+		if infos, err = parseBinOutput(req); err != nil {
 			return err
 		}
+	} else {
+		raw := req.Info
+		for len(raw.Calls) != 0 {
+			ncalls := min(len(raw.Calls), len(req.P.Calls))
+			infos = append(infos, ipc.ProgInfo{
+				Extra: raw.Extra,
+				Calls: raw.Calls[:ncalls],
+			})
+			raw.Calls = raw.Calls[ncalls:]
+		}
 	}
-	if req.Repeat != len(req.Info) {
-		return fmt.Errorf("should repeat %v times, but repeated %v\n%s",
-			req.Repeat, len(req.Info), req.Output)
+	if req.Repeat != len(infos) {
+		return fmt.Errorf("should repeat %v times, but repeated %v, prog calls %v, info calls %v\n%s",
+			req.Repeat, len(infos), req.P.Calls, len(req.Info.Calls), req.Output)
 	}
 	calls := make(map[string]bool)
-	for run, info := range req.Info {
+	for run, info := range infos {
 		for call := range info.Calls {
 			if err := checkCallResult(req, isC, run, call, info, calls); err != nil {
 				return err
@@ -509,7 +524,7 @@ func checkResult(req *RunRequest) error {
 	return nil
 }
 
-func checkCallResult(req *RunRequest, isC bool, run, call int, info *ipc.ProgInfo, calls map[string]bool) error {
+func checkCallResult(req *RunRequest, isC bool, run, call int, info ipc.ProgInfo, calls map[string]bool) error {
 	inf := info.Calls[call]
 	want := req.results.Calls[call]
 	for flag, what := range map[ipc.CallFlags]string{
@@ -571,13 +586,13 @@ func checkCallResult(req *RunRequest, isC bool, run, call int, info *ipc.ProgInf
 	return nil
 }
 
-func parseBinOutput(req *RunRequest) ([]*ipc.ProgInfo, error) {
-	var infos []*ipc.ProgInfo
+func parseBinOutput(req *RunRequest) ([]ipc.ProgInfo, error) {
+	var infos []ipc.ProgInfo
 	s := bufio.NewScanner(bytes.NewReader(req.Output))
 	re := regexp.MustCompile("^### call=([0-9]+) errno=([0-9]+)$")
 	for s.Scan() {
 		if s.Text() == "### start" {
-			infos = append(infos, &ipc.ProgInfo{Calls: make([]ipc.CallInfo, len(req.P.Calls))})
+			infos = append(infos, ipc.ProgInfo{Calls: make([]ipc.CallInfo, len(req.P.Calls))})
 		}
 		match := re.FindSubmatch(s.Bytes())
 		if match == nil {
@@ -596,7 +611,7 @@ func parseBinOutput(req *RunRequest) ([]*ipc.ProgInfo, error) {
 			return nil, fmt.Errorf("failed to parse errno %q in %q",
 				string(match[2]), s.Text())
 		}
-		info := infos[len(infos)-1]
+		info := &infos[len(infos)-1]
 		if call >= uint64(len(info.Calls)) {
 			return nil, fmt.Errorf("bad call index %v", call)
 		}
@@ -608,71 +623,3 @@ func parseBinOutput(req *RunRequest) ([]*ipc.ProgInfo, error) {
 	}
 	return infos, nil
 }
-
-func RunTest(req *RunRequest, executor string) {
-	if req.Bin != "" {
-		runTestC(req)
-		return
-	}
-	req.Cfg.Executor = executor
-	var env *ipc.Env
-	defer func() {
-		if env != nil {
-			env.Close()
-		}
-	}()
-	for run := 0; run < req.Repeat; run++ {
-		if run%2 == 0 {
-			// Recreate Env every few iterations, this allows to cover more paths.
-			if env != nil {
-				env.Close()
-				env = nil
-			}
-			var err error
-			env, err = ipc.MakeEnv(req.Cfg, 0)
-			if err != nil {
-				req.Err = fmt.Errorf("failed to create ipc env: %w", err)
-				return
-			}
-		}
-		output, info, hanged, err := env.Exec(req.Opts, req.P)
-		req.Output = append(req.Output, output...)
-		if err != nil {
-			req.Err = fmt.Errorf("run %v: failed to run: %w", run, err)
-			return
-		}
-		if hanged {
-			req.Err = fmt.Errorf("run %v: hanged", run)
-			return
-		}
-		// Detach Signal and Cover because they point into the output shmem region.
-		for i := range info.Calls {
-			info.Calls[i].Signal = append([]uint32{}, info.Calls[i].Signal...)
-			info.Calls[i].Cover = append([]uint32{}, info.Calls[i].Cover...)
-		}
-		info.Extra.Signal = append([]uint32{}, info.Extra.Signal...)
-		info.Extra.Cover = append([]uint32{}, info.Extra.Cover...)
-		req.Info = append(req.Info, info)
-	}
-}
-
-func runTestC(req *RunRequest) {
-	tmpDir, err := os.MkdirTemp("", "syz-runtest")
-	if err != nil {
-		req.Err = fmt.Errorf("failed to create temp dir: %w", err)
-		return
-	}
-	defer os.RemoveAll(tmpDir)
-	cmd := osutil.Command(req.Bin)
-	cmd.Dir = tmpDir
-	// Tell ASAN to not mess with our NONFAILING.
-	cmd.Env = append(append([]string{}, os.Environ()...), "ASAN_OPTIONS=handle_segv=0 allow_user_segv_handler=1")
-	req.Output, req.Err = osutil.Run(20*time.Second, cmd)
-	var verr *osutil.VerboseError
-	if errors.As(req.Err, &verr) {
-		// The process can legitimately do something like exit_group(1).
-		// So we ignore the error and rely on the rest of the checks (e.g. syscall return values).
-		req.Err = nil
-		req.Output = verr.Output
-	}
-}
diff --git a/pkg/runtest/run_test.go b/pkg/runtest/run_test.go
index 5cbbca865..00bf343d7 100644
--- a/pkg/runtest/run_test.go
+++ b/pkg/runtest/run_test.go
@@ -4,15 +4,18 @@
 package runtest
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
 	"testing"
+	"time"
 
 	"github.com/google/syzkaller/pkg/csource"
 	"github.com/google/syzkaller/pkg/host"
+	"github.com/google/syzkaller/pkg/ipc"
 	"github.com/google/syzkaller/pkg/osutil"
 	"github.com/google/syzkaller/pkg/testutil"
 	"github.com/google/syzkaller/prog"
@@ -84,7 +87,11 @@ func test(t *testing.T, sysTarget *targets.Target) {
 	requests := make(chan *RunRequest, 2*runtime.GOMAXPROCS(0))
 	go func() {
 		for req := range requests {
-			RunTest(req, executor)
+			if req.Bin != "" {
+				runTestC(req)
+			} else {
+				runTest(req, executor)
+			}
 			close(req.Done)
 		}
 	}()
@@ -108,6 +115,63 @@ func test(t *testing.T, sysTarget *targets.Target) {
 	}
 }
 
+func runTest(req *RunRequest, executor string) {
+	cfg := new(ipc.Config)
+	sysTarget := targets.Get(req.P.Target.OS, req.P.Target.Arch)
+	cfg.UseShmem = sysTarget.ExecutorUsesShmem
+	cfg.UseForkServer = sysTarget.ExecutorUsesForkServer
+	cfg.Timeouts = sysTarget.Timeouts(1)
+	cfg.Executor = executor
+	env, err := ipc.MakeEnv(cfg, 0)
+	if err != nil {
+		req.Err = fmt.Errorf("failed to create ipc env: %w", err)
+		return
+	}
+	defer env.Close()
+	for run := 0; run < req.Repeat; run++ {
+		if run%2 == 0 {
+			// Recreate Env every few iterations, this allows to cover more paths.
+			env.ForceRestart()
+		}
+		output, info, hanged, err := env.Exec(&req.Opts, req.P)
+		req.Output = append(req.Output, output...)
+		if err != nil {
+			req.Err = fmt.Errorf("run %v: failed to run: %w", run, err)
+			return
+		}
+		if hanged {
+			req.Err = fmt.Errorf("run %v: hanged", run)
+			return
+		}
+		if run == 0 {
+			req.Info = *info
+		} else {
+			req.Info.Calls = append(req.Info.Calls, info.Calls...)
+		}
+	}
+}
+
+func runTestC(req *RunRequest) {
+	tmpDir, err := os.MkdirTemp("", "syz-runtest")
+	if err != nil {
+		req.Err = fmt.Errorf("failed to create temp dir: %w", err)
+		return
+	}
+	defer os.RemoveAll(tmpDir)
+	cmd := osutil.Command(req.Bin)
+	cmd.Dir = tmpDir
+	// Tell ASAN to not mess with our NONFAILING.
+	cmd.Env = append(append([]string{}, os.Environ()...), "ASAN_OPTIONS=handle_segv=0 allow_user_segv_handler=1")
+	req.Output, req.Err = osutil.Run(20*time.Second, cmd)
+	var verr *osutil.VerboseError
+	if errors.As(req.Err, &verr) {
+		// The process can legitimately do something like exit_group(1).
+		// So we ignore the error and rely on the rest of the checks (e.g. syscall return values).
+		req.Err = nil
+		req.Output = verr.Output
+	}
+}
+
 func TestParsing(t *testing.T) {
 	t.Parallel()
 	// Test only one target in race mode (we have gazillion of auto-generated Linux test).
diff --git a/syz-fuzzer/fuzzer.go b/syz-fuzzer/fuzzer.go
index 70e03b7ff..40905eb94 100644
--- a/syz-fuzzer/fuzzer.go
+++ b/syz-fuzzer/fuzzer.go
@@ -55,6 +55,8 @@ type executionResult struct {
 	procID int
 	try    int
 	info   *ipc.ProgInfo
+	output []byte
+	err    string
 }
 
 // Gate size controls how deep in the log the last executed by every proc
@@ -75,13 +77,9 @@ func main() {
 		flagArch           = flag.String("arch", runtime.GOARCH, "target arch")
 		flagManager        = flag.String("manager", "", "manager rpc address")
 		flagProcs          = flag.Int("procs", 1, "number of parallel test processes")
-		flagTest           = flag.Bool("test", false, "enable image testing mode")      // used by syz-ci
-		flagRunTest        = flag.Bool("runtest", false, "enable program testing mode") // used by pkg/runtest
+		flagTest           = flag.Bool("test", false, "enable image testing mode") // used by syz-ci
 		flagPprofPort      = flag.Int("pprof_port", 0, "HTTP port for the pprof endpoint (disabled if 0)")
 		flagNetCompression = flag.Bool("net_compression", false, "use network compression for RPC calls")
-
-		// Experimental flags.
-		flagResetAccState = flag.Bool("reset_acc_state", false, "restarts executor before most executions")
 	)
 	defer tool.Init()()
 	log.Logf(0, "fuzzer started")
@@ -185,7 +183,7 @@ func main() {
 
 	log.Logf(0, "starting %v executor processes", *flagProcs)
 	for pid := 0; pid < *flagProcs; pid++ {
-		startProc(fuzzerTool, pid, config, *flagResetAccState)
+		startProc(fuzzerTool, pid, config)
 	}
 
 	checkReq.Name = *flagName
@@ -206,10 +204,6 @@ func main() {
 		log.SyzFatalf("%v", checkReq.Error)
 	}
 
-	if *flagRunTest {
-		runTest(target, manager, *flagName, executor)
-		return
-	}
 	if checkRes.CoverFilterBitmap != nil {
 		if err := osutil.WriteFile("syz-cover-bitmap", checkRes.CoverFilterBitmap); err != nil {
 			log.SyzFatalf("failed to write syz-cover-bitmap: %v", err)
@@ -340,6 +334,8 @@ func (tool *FuzzerTool) convertExecutionResult(res executionResult) rpctype.Exec
 		ID:     res.ID,
 		ProcID: res.procID,
 		Try:    res.try,
+		Output: res.output,
+		Error:  res.err,
 	}
 	if res.info != nil {
 		if res.NewSignal {
diff --git a/syz-fuzzer/proc.go b/syz-fuzzer/proc.go
index 3bcc535e3..c80de0518 100644
--- a/syz-fuzzer/proc.go
+++ b/syz-fuzzer/proc.go
@@ -4,7 +4,11 @@
 package main
 
 import (
+	"errors"
+	"fmt"
 	"math/rand"
+	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/google/syzkaller/pkg/ipc"
@@ -15,22 +19,20 @@ import (
 
 // Proc represents a single fuzzing process (executor).
 type Proc struct {
-	tool       *FuzzerTool
-	pid        int
-	env        *ipc.Env
-	resetState bool
+	tool *FuzzerTool
+	pid  int
+	env  *ipc.Env
 }
 
-func startProc(tool *FuzzerTool, pid int, config *ipc.Config, resetState bool) {
+func startProc(tool *FuzzerTool, pid int, config *ipc.Config) {
 	env, err := ipc.MakeEnv(config, pid)
 	if err != nil {
 		log.SyzFatalf("failed to create env: %v", err)
 	}
 	proc := &Proc{
-		tool:       tool,
-		pid:        pid,
-		env:        env,
-		resetState: resetState,
+		tool: tool,
+		pid:  pid,
+		env:  env,
 	}
 	go proc.loop()
 }
@@ -41,19 +43,36 @@ func (proc *Proc) loop() {
 		req := proc.nextRequest()
 		// Do not let too much state accumulate.
 		const restartIn = 600
-		if req.ExecOpts.ExecFlags&(ipc.FlagCollectSignal|ipc.FlagCollectCover|ipc.FlagCollectComps) != 0 &&
-			(proc.resetState || rnd.Intn(restartIn) == 0) {
+		if (req.ExecOpts.ExecFlags&(ipc.FlagCollectSignal|ipc.FlagCollectCover|ipc.FlagCollectComps) != 0 &&
+			rnd.Intn(restartIn) == 0) || req.ResetState {
 			proc.env.ForceRestart()
 		}
-		info, try := proc.execute(req)
-		// Let's perform signal filtering in a separate thread to get the most
-		// exec/sec out of a syz-executor instance.
-		proc.tool.results <- executionResult{
+		info, output, err, try := proc.execute(req)
+		res := executionResult{
 			ExecutionRequest: req,
 			procID:           proc.pid,
 			try:              try,
 			info:             info,
+			output:           output,
+			err:              err,
+		}
+		for i := 1; i < req.Repeat && res.err == ""; i++ {
+			// Recreate Env every few iterations, this allows to cover more paths.
+			if i%2 == 0 && !req.IsBinary {
+				proc.env.ForceRestart()
+			}
+			info, output, err, _ := proc.execute(req)
+			if res.info == nil {
+				res.info = info
+			} else {
+				res.info.Calls = append(res.info.Calls, info.Calls...)
+			}
+			res.output = append(res.output, output...)
+			res.err = err
 		}
+		// Let's perform signal filtering in a separate thread to get the most
+		// exec/sec out of a syz-executor instance.
+		proc.tool.results <- res
 	}
 }
 
@@ -72,7 +91,23 @@ func (proc *Proc) nextRequest() rpctype.ExecutionRequest {
 	return req
 }
 
-func (proc *Proc) execute(req rpctype.ExecutionRequest) (*ipc.ProgInfo, int) {
+func (proc *Proc) execute(req rpctype.ExecutionRequest) (info *ipc.ProgInfo, output []byte, errStr string, try int) {
+	var err error
+	if req.IsBinary {
+		output, err = executeBinary(req)
+	} else {
+		info, output, try, err = proc.executeProgram(req)
+	}
+	if !req.ReturnOutput {
+		output = nil
+	}
+	if err != nil {
+		errStr = err.Error()
+	}
+	return
+}
+
+func (proc *Proc) executeProgram(req rpctype.ExecutionRequest) (*ipc.ProgInfo, []byte, int, error) {
 	for try := 0; ; try++ {
 		var output []byte
 		var info *ipc.ProgInfo
@@ -86,11 +121,14 @@ func (proc *Proc) execute(req rpctype.ExecutionRequest) (*ipc.ProgInfo, int) {
 			proc.tool.startExecutingCall(req.ID, proc.pid, try)
 			output, info, hanged, err = proc.env.ExecProg(&req.ExecOpts, req.ProgData)
 			proc.tool.gate.Leave(ticket)
-			if err == nil {
-				log.Logf(2, "result hanged=%v: %s", hanged, output)
-				return info, try
+			log.Logf(2, "result hanged=%v err=%v: %s", hanged, err, output)
+			if hanged && err == nil && req.ReturnError {
+				err = errors.New("hanged")
 			}
 		}
+		if err == nil || req.ReturnError {
+			return info, output, try, err
+		}
 		log.Logf(4, "fuzzer detected executor failure='%v', retrying #%d", err, try+1)
 		if try > 10 {
 			log.SyzFatalf("executor %v failed %v times: %v\n%s", proc.pid, try, err, output)
@@ -99,3 +137,27 @@ func (proc *Proc) execute(req rpctype.ExecutionRequest) (*ipc.ProgInfo, int) {
 		}
 	}
 }
+
+func executeBinary(req rpctype.ExecutionRequest) ([]byte, error) {
+	tmp, err := os.MkdirTemp("", "syz-runtest")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temp dir: %w", err)
+	}
+	defer os.RemoveAll(tmp)
+	bin := filepath.Join(tmp, "syz-executor")
+	if err := os.WriteFile(bin, req.ProgData, 0777); err != nil {
+		return nil, fmt.Errorf("failed to write binary: %w", err)
+	}
+	cmd := osutil.Command(bin)
+	cmd.Dir = tmp
+	// Tell ASAN to not mess with our NONFAILING.
+	cmd.Env = append(append([]string{}, os.Environ()...), "ASAN_OPTIONS=handle_segv=0 allow_user_segv_handler=1")
+	output, err := osutil.Run(20*time.Second, cmd)
+	var verr *osutil.VerboseError
+	if errors.As(err, &verr) {
+		// The process can legitimately do something like exit_group(1).
+		// So we ignore the error and rely on the rest of the checks (e.g. syscall return values).
+		return verr.Output, nil
+	}
+	return output, err
+}
diff --git a/syz-fuzzer/testing.go b/syz-fuzzer/testing.go
index f810e3e1b..8bfecf5a1 100644
--- a/syz-fuzzer/testing.go
+++ b/syz-fuzzer/testing.go
@@ -15,7 +15,6 @@ import (
 	"github.com/google/syzkaller/pkg/log"
 	"github.com/google/syzkaller/pkg/osutil"
 	"github.com/google/syzkaller/pkg/rpctype"
-	"github.com/google/syzkaller/pkg/runtest"
 	"github.com/google/syzkaller/prog"
 )
 
@@ -47,64 +46,6 @@ func testImage(hostAddr string, args *checkArgs) {
 	}
 }
 
-func runTest(target *prog.Target, manager *rpctype.RPCClient, name, executor string) {
-	pollReq := &rpctype.RunTestPollReq{Name: name}
-	for {
-		req := new(rpctype.RunTestPollRes)
-		if err := manager.Call("Manager.Poll", pollReq, req); err != nil {
-			log.SyzFatalf("Manager.Poll call failed: %v", err)
-		}
-		if len(req.Bin) == 0 && len(req.Prog) == 0 {
-			return
-		}
-		test := convertTestReq(target, req)
-		if test.Err == nil {
-			runtest.RunTest(test, executor)
-		}
-		reply := &rpctype.RunTestDoneArgs{
-			Name:   name,
-			ID:     req.ID,
-			Output: test.Output,
-			Info:   test.Info,
-		}
-		if test.Err != nil {
-			reply.Error = test.Err.Error()
-		}
-		if err := manager.Call("Manager.Done", reply, nil); err != nil {
-			log.SyzFatalf("Manager.Done call failed: %v", err)
-		}
-	}
-}
-
-func convertTestReq(target *prog.Target, req *rpctype.RunTestPollRes) *runtest.RunRequest {
-	test := &runtest.RunRequest{
-		Cfg:    req.Cfg,
-		Opts:   req.Opts,
-		Repeat: req.Repeat,
-	}
-	if len(req.Bin) != 0 {
-		bin, err := osutil.TempFile("syz-runtest")
-		if err != nil {
-			test.Err = err
-			return test
-		}
-		if err := osutil.WriteExecFile(bin, req.Bin); err != nil {
-			test.Err = err
-			return test
-		}
-		test.Bin = bin
-	}
-	if len(req.Prog) != 0 {
-		p, err := target.Deserialize(req.Prog, prog.NonStrict)
-		if err != nil {
-			test.Err = err
-			return test
-		}
-		test.P = p
-	}
-	return test
-}
-
 func checkMachineHeartbeats(done chan bool) {
 	ticker := time.NewTicker(3 * time.Second)
 	defer ticker.Stop()
diff --git a/syz-manager/manager.go b/syz-manager/manager.go
index 3e77e7885..469ac38c4 100644
--- a/syz-manager/manager.go
+++ b/syz-manager/manager.go
@@ -810,12 +810,10 @@ func (mgr *Manager) runInstanceInner(index int, instanceName string, injectLog <
 		Cover:     mgr.cfg.Cover,
 		Debug:     *flagDebug,
 		Test:      false,
-		Runtest:   false,
 		Optional: &instance.OptionalFuzzerArgs{
 			Slowdown:       mgr.cfg.Timeouts.Slowdown,
 			SandboxArg:     mgr.cfg.SandboxArg,
 			PprofPort:      inst.PprofPort(),
-			ResetAccState:  mgr.cfg.Experimental.ResetAccState,
 			NetCompression: mgr.netCompression,
 		},
 	}
diff --git a/syz-manager/rpc.go b/syz-manager/rpc.go
index d3afd780f..fab496d26 100644
--- a/syz-manager/rpc.go
+++ b/syz-manager/rpc.go
@@ -570,6 +570,7 @@ func (serv *RPCServer) newRequest(runner *Runner, req *fuzzer.Request) (rpctype.
 		NewSignal:        req.NeedSignal == fuzzer.NewSignal,
 		SignalFilter:     signalFilter,
 		SignalFilterCall: req.SignalFilterCall,
+		ResetState:       serv.cfg.Experimental.ResetAccState,
 	}, true
 }
 
diff --git a/tools/syz-runtest/empty.go b/tools/syz-runtest/empty.go
deleted file mode 100644
index 588c0c9e8..000000000
--- a/tools/syz-runtest/empty.go
+++ /dev/null
@@ -1,6 +0,0 @@
-// Copyright 2024 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package main
-
-func main() {}
diff --git a/tools/syz-runtest/runtest.go b/tools/syz-runtest/runtest.go
index 5849131eb..82a1cc30e 100644
--- a/tools/syz-runtest/runtest.go
+++ b/tools/syz-runtest/runtest.go
@@ -1,10 +1,6 @@
 // Copyright 2018 syzkaller project authors. All rights reserved.
 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
 
-// This is broken for now.
-
-//go:build ignore
-
 // Runtest runs syzkaller test programs in sys/*/test/*. Start as:
 // $ syz-runtest -config manager.config
 // Also see pkg/runtest docs.
@@ -19,6 +15,7 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/google/syzkaller/pkg/host"
@@ -61,13 +58,16 @@ func main() {
 		checker:            vminfo.New(cfg),
 		reporter:           reporter,
 		debug:              *flagDebug,
-		requests:           make(chan *runtest.RunRequest, 2*vmPool.Count()),
+		requests:           make(chan *runtest.RunRequest, 4*vmPool.Count()*cfg.Procs),
 		checkResultC:       make(chan *rpctype.CheckArgs, 1),
+		checkProgsDone:     make(chan bool),
 		checkFeaturesReady: make(chan bool),
 		vmStop:             make(chan bool),
-		reqMap:             make(map[int]*runtest.RunRequest),
-		lastReq:            make(map[string]int),
+		reqMap:             make(map[int64]*runtest.RunRequest),
+		pending:            make(map[string]map[int64]bool),
 	}
+	mgr.checkFiles, mgr.checkProgs = mgr.checker.StartCheck()
+	mgr.needCheckResults = len(mgr.checkProgs)
 	s, err := rpctype.NewRPCServer(cfg.RPC, "Manager", mgr, false)
 	if err != nil {
 		log.Fatalf("failed to create rpc server: %v", err)
@@ -77,12 +77,13 @@ func main() {
 	var wg sync.WaitGroup
 	wg.Add(vmPool.Count())
 	fmt.Printf("booting VMs...\n")
+	var nameSeq atomic.Uint64
 	for i := 0; i < vmPool.Count(); i++ {
 		i := i
 		go func() {
 			defer wg.Done()
-			name := fmt.Sprintf("vm-%v", i)
 			for {
+				name := fmt.Sprintf("vm-%v", nameSeq.Add(1))
 				rep, err := mgr.boot(name, i)
 				if err != nil {
 					log.Fatal(err)
@@ -90,7 +91,7 @@ func main() {
 				if rep == nil {
 					return
 				}
-				if err := mgr.finishRequest(name, rep); err != nil {
+				if err := mgr.finishRequests(name, rep); err != nil {
 					log.Fatal(err)
 				}
 			}
@@ -98,8 +99,10 @@ func main() {
 	}
 	checkResult := <-mgr.checkResultC
 	mgr.checkFeatures = checkResult.Features
+	mgr.checkFilesInfo = checkResult.Files
 	close(mgr.checkFeaturesReady)
-	calls, _, err := mgr.checker.Check(checkResult.Files, checkResult.CheckProgs)
+	<-mgr.checkProgsDone
+	calls, _, err := mgr.checker.FinishCheck(mgr.checkFilesInfo, mgr.checkResults)
 	if err != nil {
 		log.Fatalf("failed to detect enabled syscalls: %v", err)
 	}
@@ -142,6 +145,12 @@ type Manager struct {
 	cfg                *mgrconfig.Config
 	vmPool             *vm.Pool
 	checker            *vminfo.Checker
+	checkFiles         []string
+	checkFilesInfo     []host.FileInfo
+	checkProgs         []rpctype.ExecutionRequest
+	checkResults       []rpctype.ExecutionResult
+	needCheckResults   int
+	checkProgsDone     chan bool
 	reporter           *report.Reporter
 	requests           chan *runtest.RunRequest
 	checkFeatures      *host.Features
@@ -152,9 +161,9 @@ type Manager struct {
 	debug              bool
 
 	reqMu   sync.Mutex
-	reqSeq  int
-	reqMap  map[int]*runtest.RunRequest
-	lastReq map[string]int
+	reqSeq  int64
+	reqMap  map[int64]*runtest.RunRequest
+	pending map[string]map[int64]bool
 }
 
 func (mgr *Manager) boot(name string, index int) (*report.Report, error) {
@@ -191,12 +200,11 @@ func (mgr *Manager) boot(name string, index int) (*report.Report, error) {
 		Arch:      mgr.cfg.TargetArch,
 		FwdAddr:   fwdAddr,
 		Sandbox:   mgr.cfg.Sandbox,
-		Procs:     mgr.cfg.Procs,
+		Procs:     1,
 		Verbosity: 0,
 		Cover:     mgr.cfg.Cover,
 		Debug:     mgr.debug,
 		Test:      false,
-		Runtest:   true,
 		Optional: &instance.OptionalFuzzerArgs{
 			Slowdown:   mgr.cfg.Timeouts.Slowdown,
 			SandboxArg: mgr.cfg.SandboxArg,
@@ -210,22 +218,23 @@ func (mgr *Manager) boot(name string, index int) (*report.Report, error) {
 	return rep, nil
 }
 
-func (mgr *Manager) finishRequest(name string, rep *report.Report) error {
+func (mgr *Manager) finishRequests(name string, rep *report.Report) error {
 	mgr.reqMu.Lock()
 	defer mgr.reqMu.Unlock()
-	lastReq := mgr.lastReq[name]
-	req := mgr.reqMap[lastReq]
-	if lastReq == 0 || req == nil {
-		return fmt.Errorf("vm crash: %v\n%s\n%s", rep.Title, rep.Report, rep.Output)
-	}
-	delete(mgr.reqMap, lastReq)
-	delete(mgr.lastReq, name)
-	req.Err = fmt.Errorf("%v", rep.Title)
-	req.Output = rep.Report
-	if len(req.Output) == 0 {
-		req.Output = rep.Output
+	for id := range mgr.pending[name] {
+		req := mgr.reqMap[id]
+		if req == nil {
+			return fmt.Errorf("vm crash: %v\n%s\n%s", rep.Title, rep.Report, rep.Output)
+		}
+		delete(mgr.reqMap, id)
+		req.Err = fmt.Errorf("%v", rep.Title)
+		req.Output = rep.Report
+		if len(req.Output) == 0 {
+			req.Output = rep.Output
+		}
+		close(req.Done)
 	}
-	close(req.Done)
+	delete(mgr.pending, name)
 	return nil
 }
 
@@ -234,10 +243,8 @@ func (mgr *Manager) Connect(a *rpctype.ConnectArgs, r *rpctype.ConnectRes) error
 	case <-mgr.checkFeaturesReady:
 		r.Features = mgr.checkFeatures
 	default:
-		infoFiles, checkFiles, checkProgs := mgr.checker.RequiredThings()
-		r.ReadFiles = append(infoFiles, checkFiles...)
+		r.ReadFiles = append(mgr.checker.RequiredFiles(), mgr.checkFiles...)
 		r.ReadGlobs = mgr.cfg.Target.RequiredGlobs()
-		r.CheckProgs = checkProgs
 	}
 	return nil
 }
@@ -253,53 +260,83 @@ func (mgr *Manager) Check(a *rpctype.CheckArgs, r *rpctype.CheckRes) error {
 	return nil
 }
 
-func (mgr *Manager) Poll(a *rpctype.RunTestPollReq, r *rpctype.RunTestPollRes) error {
-	req := <-mgr.requests
-	if req == nil {
+func (mgr *Manager) ExchangeInfo(a *rpctype.ExchangeInfoRequest, r *rpctype.ExchangeInfoReply) error {
+	mgr.reqMu.Lock()
+	defer mgr.reqMu.Unlock()
+
+	select {
+	case <-mgr.checkProgsDone:
+	default:
+		mgr.checkResults = append(mgr.checkResults, a.Results...)
+		if len(mgr.checkResults) < mgr.needCheckResults {
+			numRequests := min(len(mgr.checkProgs), a.NeedProgs)
+			r.Requests = mgr.checkProgs[:numRequests]
+			mgr.checkProgs = mgr.checkProgs[numRequests:]
+		} else {
+			close(mgr.checkProgsDone)
+		}
 		return nil
 	}
-	mgr.reqMu.Lock()
-	if mgr.lastReq[a.Name] != 0 {
-		log.Fatalf("double poll req from %v", a.Name)
+
+	if mgr.pending[a.Name] == nil {
+		mgr.pending[a.Name] = make(map[int64]bool)
 	}
-	mgr.reqSeq++
-	r.ID = mgr.reqSeq
-	mgr.reqMap[mgr.reqSeq] = req
-	mgr.lastReq[a.Name] = mgr.reqSeq
-	mgr.reqMu.Unlock()
-	if req.Bin != "" {
-		data, err := os.ReadFile(req.Bin)
+	for _, res := range a.Results {
+		if !mgr.pending[a.Name][res.ID] {
+			log.Fatalf("runner %v wasn't executing request %v", a.Name, res.ID)
+		}
+		delete(mgr.pending[a.Name], res.ID)
+		req := mgr.reqMap[res.ID]
+		if req == nil {
+			log.Fatalf("request %v does not exist", res.ID)
+		}
+		delete(mgr.reqMap, res.ID)
+		if req == nil {
+			log.Fatalf("got done request for unknown id %v", res.ID)
+		}
+		req.Output = res.Output
+		req.Info = res.Info
+		if res.Error != "" {
+			req.Err = errors.New(res.Error)
+		}
+		close(req.Done)
+	}
+	for i := 0; i < a.NeedProgs; i++ {
+		var req *runtest.RunRequest
+		select {
+		case req = <-mgr.requests:
+		default:
+		}
+		if req == nil {
+			break
+		}
+		mgr.reqSeq++
+		mgr.reqMap[mgr.reqSeq] = req
+		mgr.pending[a.Name][mgr.reqSeq] = true
+		var progData []byte
+		var err error
+		if req.Bin != "" {
+			progData, err = os.ReadFile(req.Bin)
+		} else {
+			progData, err = req.P.SerializeForExec()
+		}
 		if err != nil {
-			log.Fatalf("failed to read bin file: %v", err)
+			log.Fatal(err)
 		}
-		r.Bin = data
-		return nil
+		r.Requests = append(r.Requests, rpctype.ExecutionRequest{
+			ID:           mgr.reqSeq,
+			ProgData:     progData,
+			ExecOpts:     req.Opts,
+			IsBinary:     req.Bin != "",
+			ResetState:   req.Bin == "",
+			ReturnOutput: true,
+			ReturnError:  true,
+			Repeat:       req.Repeat,
+		})
 	}
-	r.Prog = req.P.Serialize()
-	r.Cfg = req.Cfg
-	r.Opts = req.Opts
-	r.Repeat = req.Repeat
 	return nil
 }
 
-func (mgr *Manager) Done(a *rpctype.RunTestDoneArgs, r *int) error {
-	mgr.reqMu.Lock()
-	lastReq := mgr.lastReq[a.Name]
-	if lastReq != a.ID {
-		log.Fatalf("wrong done id %v from %v", a.ID, a.Name)
-	}
-	req := mgr.reqMap[a.ID]
-	delete(mgr.reqMap, a.ID)
-	delete(mgr.lastReq, a.Name)
-	mgr.reqMu.Unlock()
-	if req == nil {
-		log.Fatalf("got done request for unknown id %v", a.ID)
-	}
-	req.Output = a.Output
-	req.Info = a.Info
-	if a.Error != "" {
-		req.Err = errors.New(a.Error)
-	}
-	close(req.Done)
+func (mgr *Manager) StartExecuting(a *rpctype.ExecutingRequest, r *int) error {
 	return nil
 }