tools/syz-runtest: switch to the generic program execution

syz-runtest effectively implemented the same execute program/return result
mechanism we use now for normal fuzzing.
So extend the general mechanism to allow collecting output/errors,
repeating program, and executing a precompiled binary as a test.
And switch syz-runtest to the general mechanism.
This removes another chunk of code from syz-fuzzer.

4 files changed, 141 insertions, 145 deletions

diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go
index 7f1ca4dbb..3e59209e2 100644
--- a/pkg/instance/instance.go
+++ b/pkg/instance/instance.go
@@ -459,7 +459,6 @@ type OptionalFuzzerArgs struct {
 	Slowdown       int
 	SandboxArg     int
 	PprofPort      int
-	ResetAccState  bool
 	NetCompression bool
 }
 
@@ -476,7 +475,6 @@ type FuzzerCmdArgs struct {
 	Cover     bool
 	Debug     bool
 	Test      bool
-	Runtest   bool
 	Optional  *OptionalFuzzerArgs
 }
 
@@ -488,10 +486,6 @@ func FuzzerCmd(args *FuzzerCmdArgs) string {
 		// because old execprog does not have os flag.
 		osArg = " -os=" + args.OS
 	}
-	runtestArg := ""
-	if args.Runtest {
-		runtestArg = " -runtest"
-	}
 	verbosityArg := ""
 	if args.Verbosity != 0 {
 		verbosityArg = fmt.Sprintf(" -vv=%v", args.Verbosity)
@@ -502,15 +496,14 @@ func FuzzerCmd(args *FuzzerCmdArgs) string {
 			{Name: "slowdown", Value: fmt.Sprint(args.Optional.Slowdown)},
 			{Name: "sandbox_arg", Value: fmt.Sprint(args.Optional.SandboxArg)},
 			{Name: "pprof_port", Value: fmt.Sprint(args.Optional.PprofPort)},
-			{Name: "reset_acc_state", Value: fmt.Sprint(args.Optional.ResetAccState)},
 			{Name: "net_compression", Value: fmt.Sprint(args.Optional.NetCompression)},
 		}
 		optionalArg = " " + tool.OptionalFlags(flags)
 	}
 	return fmt.Sprintf("%v -executor=%v -name=%v -arch=%v%v -manager=%v -sandbox=%v"+
-		" -procs=%v -cover=%v -debug=%v -test=%v%v%v%v",
+		" -procs=%v -cover=%v -debug=%v -test=%v%v%v",
 		args.Fuzzer, args.Executor, args.Name, args.Arch, osArg, args.FwdAddr, args.Sandbox,
-		args.Procs, args.Cover, args.Debug, args.Test, runtestArg, verbosityArg, optionalArg)
+		args.Procs, args.Cover, args.Debug, args.Test, verbosityArg, optionalArg)
 }
 
 func OldFuzzerCmd(fuzzer, executor, name, OS, arch, fwdAddr, sandbox string, sandboxArg, procs int,
@@ -521,7 +514,7 @@ func OldFuzzerCmd(fuzzer, executor, name, OS, arch, fwdAddr, sandbox string, san
 	}
 	return FuzzerCmd(&FuzzerCmdArgs{Fuzzer: fuzzer, Executor: executor, Name: name,
 		OS: OS, Arch: arch, FwdAddr: fwdAddr, Sandbox: sandbox,
-		Procs: procs, Verbosity: 0, Cover: cover, Debug: false, Test: test, Runtest: false,
+		Procs: procs, Verbosity: 0, Cover: cover, Debug: false, Test: test,
 		Optional: optional})
 }
 
diff --git a/pkg/rpctype/rpctype.go b/pkg/rpctype/rpctype.go
index 64caf6a04..0c26c65e8 100644
--- a/pkg/rpctype/rpctype.go
+++ b/pkg/rpctype/rpctype.go
@@ -17,12 +17,23 @@ import (
 // ExecutionRequest describes the task of executing a particular program.
 // Corresponds to Fuzzer.Request.
 type ExecutionRequest struct {
-	ID               int64
-	ProgData         []byte
-	ExecOpts         ipc.ExecOpts
-	NewSignal        bool
+	ID        int64
+	ProgData  []byte
+	ExecOpts  ipc.ExecOpts
+	NewSignal bool
+	// If set, ProgData contains compiled executable binary
+	// that needs to be written to disk and executed.
+	IsBinary bool
+	// If set, fully reset executor state befor executing the test.
+	ResetState bool
+	// If set, collect program output and return in ExecutionResult.Output.
+	ReturnOutput bool
+	// If set, don't fail on program failures, instead return the error in ExecutionResult.Error.
+	ReturnError      bool
 	SignalFilter     signal.Signal
 	SignalFilterCall int
+	// Repeat the program that many times (0 means 1).
+	Repeat int
 }
 
 // ExecutionResult is sent after ExecutionRequest is completed.
@@ -31,6 +42,8 @@ type ExecutionResult struct {
 	ProcID int
 	Try    int
 	Info   ipc.ProgInfo
+	Output []byte
+	Error  string
 }
 
 // ExchangeInfoRequest is periodically sent by syz-fuzzer to syz-manager.
@@ -198,24 +211,3 @@ type HubInput struct {
 	Domain string
 	Prog   []byte
 }
-
-type RunTestPollReq struct {
-	Name string
-}
-
-type RunTestPollRes struct {
-	ID     int
-	Bin    []byte
-	Prog   []byte
-	Cfg    *ipc.Config
-	Opts   *ipc.ExecOpts
-	Repeat int
-}
-
-type RunTestDoneArgs struct {
-	Name   string
-	ID     int
-	Output []byte
-	Info   []*ipc.ProgInfo
-	Error  string
-}
diff --git a/pkg/runtest/run.go b/pkg/runtest/run.go
index 5c00bf982..796fb6040 100644
--- a/pkg/runtest/run.go
+++ b/pkg/runtest/run.go
@@ -13,7 +13,6 @@ package runtest
 import (
 	"bufio"
 	"bytes"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -22,12 +21,10 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/google/syzkaller/pkg/csource"
 	"github.com/google/syzkaller/pkg/host"
 	"github.com/google/syzkaller/pkg/ipc"
-	"github.com/google/syzkaller/pkg/osutil"
 	"github.com/google/syzkaller/prog"
 	"github.com/google/syzkaller/sys/targets"
 )
@@ -35,19 +32,19 @@ import (
 type RunRequest struct {
 	Bin    string
 	P      *prog.Prog
-	Cfg    *ipc.Config
-	Opts   *ipc.ExecOpts
+	Opts   ipc.ExecOpts
 	Repeat int
 
 	Done   chan struct{}
 	Output []byte
-	Info   []*ipc.ProgInfo
+	Info   ipc.ProgInfo
 	Err    error
 
-	results *ipc.ProgInfo
-	name    string
-	broken  string
-	skip    string
+	finished chan struct{}
+	results  *ipc.ProgInfo
+	name     string
+	broken   string
+	skip     string
 }
 
 type Context struct {
@@ -78,19 +75,15 @@ func (ctx *Context) Run() error {
 		defer close(progs)
 		errc <- ctx.generatePrograms(progs)
 	}()
-	var ok, fail, broken, skip int
+	var requests []*RunRequest
 	for req := range progs {
-		result := ""
-		verbose := false
-		if req.broken != "" {
-			broken++
-			result = fmt.Sprintf("BROKEN (%v)", req.broken)
-			verbose = true
-		} else if req.skip != "" {
-			skip++
-			result = fmt.Sprintf("SKIP (%v)", req.skip)
-			verbose = true
-		} else {
+		req := req
+		requests = append(requests, req)
+		if req.broken != "" || req.skip != "" {
+			continue
+		}
+		req.finished = make(chan struct{})
+		go func() {
 			// The tests depend on timings and may be flaky, esp on overloaded/slow machines.
 			// We don't want to fix this by significantly bumping all timeouts,
 			// because if a program fails all the time with the default timeouts,
@@ -102,7 +95,7 @@ func (ctx *Context) Run() error {
 			var resultErr error
 			for try, failed := 0, 0; try < ctx.Retries; try++ {
 				req.Output = nil
-				req.Info = nil
+				req.Info = ipc.ProgInfo{}
 				req.Done = make(chan struct{})
 				ctx.Requests <- req
 				<-req.Done
@@ -122,6 +115,23 @@ func (ctx *Context) Run() error {
 			if req.Err == nil {
 				req.Err = resultErr
 			}
+			close(req.finished)
+		}()
+	}
+	var ok, fail, broken, skip int
+	for _, req := range requests {
+		result := ""
+		verbose := false
+		if req.broken != "" {
+			broken++
+			result = fmt.Sprintf("BROKEN (%v)", req.broken)
+			verbose = true
+		} else if req.skip != "" {
+			skip++
+			result = fmt.Sprintf("SKIP (%v)", req.skip)
+			verbose = true
+		} else {
+			<-req.finished
 			if req.Err != nil {
 				fail++
 				result = fmt.Sprintf("FAIL: %v",
@@ -400,12 +410,7 @@ func match(props, requires map[string]bool) bool {
 }
 
 func (ctx *Context) createSyzTest(p *prog.Prog, sandbox string, threaded, cov bool, times int) (*RunRequest, error) {
-	sysTarget := targets.Get(p.Target.OS, p.Target.Arch)
-	cfg := new(ipc.Config)
-	opts := new(ipc.ExecOpts)
-	cfg.UseShmem = sysTarget.ExecutorUsesShmem
-	cfg.UseForkServer = sysTarget.ExecutorUsesForkServer
-	cfg.Timeouts = sysTarget.Timeouts(1)
+	var opts ipc.ExecOpts
 	sandboxFlags, err := ipc.SandboxToFlags(sandbox)
 	if err != nil {
 		return nil, err
@@ -425,7 +430,6 @@ func (ctx *Context) createSyzTest(p *prog.Prog, sandbox string, threaded, cov bo
 	}
 	req := &RunRequest{
 		P:      p,
-		Cfg:    cfg,
 		Opts:   opts,
 		Repeat: times,
 	}
@@ -478,7 +482,7 @@ func (ctx *Context) createCTest(p *prog.Prog, sandbox string, threaded bool, tim
 	req := &RunRequest{
 		P:   p,
 		Bin: bin,
-		Opts: &ipc.ExecOpts{
+		Opts: ipc.ExecOpts{
 			ExecFlags: ipcFlags,
 		},
 		Repeat: times,
@@ -487,19 +491,30 @@ func (ctx *Context) createCTest(p *prog.Prog, sandbox string, threaded bool, tim
 }
 
 func checkResult(req *RunRequest) error {
+	var infos []ipc.ProgInfo
 	isC := req.Bin != ""
 	if isC {
 		var err error
-		if req.Info, err = parseBinOutput(req); err != nil {
+		if infos, err = parseBinOutput(req); err != nil {
 			return err
 		}
+	} else {
+		raw := req.Info
+		for len(raw.Calls) != 0 {
+			ncalls := min(len(raw.Calls), len(req.P.Calls))
+			infos = append(infos, ipc.ProgInfo{
+				Extra: raw.Extra,
+				Calls: raw.Calls[:ncalls],
+			})
+			raw.Calls = raw.Calls[ncalls:]
+		}
 	}
-	if req.Repeat != len(req.Info) {
-		return fmt.Errorf("should repeat %v times, but repeated %v\n%s",
-			req.Repeat, len(req.Info), req.Output)
+	if req.Repeat != len(infos) {
+		return fmt.Errorf("should repeat %v times, but repeated %v, prog calls %v, info calls %v\n%s",
+			req.Repeat, len(infos), req.P.Calls, len(req.Info.Calls), req.Output)
 	}
 	calls := make(map[string]bool)
-	for run, info := range req.Info {
+	for run, info := range infos {
 		for call := range info.Calls {
 			if err := checkCallResult(req, isC, run, call, info, calls); err != nil {
 				return err
@@ -509,7 +524,7 @@ func checkResult(req *RunRequest) error {
 	return nil
 }
 
-func checkCallResult(req *RunRequest, isC bool, run, call int, info *ipc.ProgInfo, calls map[string]bool) error {
+func checkCallResult(req *RunRequest, isC bool, run, call int, info ipc.ProgInfo, calls map[string]bool) error {
 	inf := info.Calls[call]
 	want := req.results.Calls[call]
 	for flag, what := range map[ipc.CallFlags]string{
@@ -571,13 +586,13 @@ func checkCallResult(req *RunRequest, isC bool, run, call int, info *ipc.ProgInf
 	return nil
 }
 
-func parseBinOutput(req *RunRequest) ([]*ipc.ProgInfo, error) {
-	var infos []*ipc.ProgInfo
+func parseBinOutput(req *RunRequest) ([]ipc.ProgInfo, error) {
+	var infos []ipc.ProgInfo
 	s := bufio.NewScanner(bytes.NewReader(req.Output))
 	re := regexp.MustCompile("^### call=([0-9]+) errno=([0-9]+)$")
 	for s.Scan() {
 		if s.Text() == "### start" {
-			infos = append(infos, &ipc.ProgInfo{Calls: make([]ipc.CallInfo, len(req.P.Calls))})
+			infos = append(infos, ipc.ProgInfo{Calls: make([]ipc.CallInfo, len(req.P.Calls))})
 		}
 		match := re.FindSubmatch(s.Bytes())
 		if match == nil {
@@ -596,7 +611,7 @@ func parseBinOutput(req *RunRequest) ([]*ipc.ProgInfo, error) {
 			return nil, fmt.Errorf("failed to parse errno %q in %q",
 				string(match[2]), s.Text())
 		}
-		info := infos[len(infos)-1]
+		info := &infos[len(infos)-1]
 		if call >= uint64(len(info.Calls)) {
 			return nil, fmt.Errorf("bad call index %v", call)
 		}
@@ -608,71 +623,3 @@ func parseBinOutput(req *RunRequest) ([]*ipc.ProgInfo, error) {
 	}
 	return infos, nil
 }
-
-func RunTest(req *RunRequest, executor string) {
-	if req.Bin != "" {
-		runTestC(req)
-		return
-	}
-	req.Cfg.Executor = executor
-	var env *ipc.Env
-	defer func() {
-		if env != nil {
-			env.Close()
-		}
-	}()
-	for run := 0; run < req.Repeat; run++ {
-		if run%2 == 0 {
-			// Recreate Env every few iterations, this allows to cover more paths.
-			if env != nil {
-				env.Close()
-				env = nil
-			}
-			var err error
-			env, err = ipc.MakeEnv(req.Cfg, 0)
-			if err != nil {
-				req.Err = fmt.Errorf("failed to create ipc env: %w", err)
-				return
-			}
-		}
-		output, info, hanged, err := env.Exec(req.Opts, req.P)
-		req.Output = append(req.Output, output...)
-		if err != nil {
-			req.Err = fmt.Errorf("run %v: failed to run: %w", run, err)
-			return
-		}
-		if hanged {
-			req.Err = fmt.Errorf("run %v: hanged", run)
-			return
-		}
-		// Detach Signal and Cover because they point into the output shmem region.
-		for i := range info.Calls {
-			info.Calls[i].Signal = append([]uint32{}, info.Calls[i].Signal...)
-			info.Calls[i].Cover = append([]uint32{}, info.Calls[i].Cover...)
-		}
-		info.Extra.Signal = append([]uint32{}, info.Extra.Signal...)
-		info.Extra.Cover = append([]uint32{}, info.Extra.Cover...)
-		req.Info = append(req.Info, info)
-	}
-}
-
-func runTestC(req *RunRequest) {
-	tmpDir, err := os.MkdirTemp("", "syz-runtest")
-	if err != nil {
-		req.Err = fmt.Errorf("failed to create temp dir: %w", err)
-		return
-	}
-	defer os.RemoveAll(tmpDir)
-	cmd := osutil.Command(req.Bin)
-	cmd.Dir = tmpDir
-	// Tell ASAN to not mess with our NONFAILING.
-	cmd.Env = append(append([]string{}, os.Environ()...), "ASAN_OPTIONS=handle_segv=0 allow_user_segv_handler=1")
-	req.Output, req.Err = osutil.Run(20*time.Second, cmd)
-	var verr *osutil.VerboseError
-	if errors.As(req.Err, &verr) {
-		// The process can legitimately do something like exit_group(1).
-		// So we ignore the error and rely on the rest of the checks (e.g. syscall return values).
-		req.Err = nil
-		req.Output = verr.Output
-	}
-}
diff --git a/pkg/runtest/run_test.go b/pkg/runtest/run_test.go
index 5cbbca865..00bf343d7 100644
--- a/pkg/runtest/run_test.go
+++ b/pkg/runtest/run_test.go
@@ -4,15 +4,18 @@
 package runtest
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
 	"testing"
+	"time"
 
 	"github.com/google/syzkaller/pkg/csource"
 	"github.com/google/syzkaller/pkg/host"
+	"github.com/google/syzkaller/pkg/ipc"
 	"github.com/google/syzkaller/pkg/osutil"
 	"github.com/google/syzkaller/pkg/testutil"
 	"github.com/google/syzkaller/prog"
@@ -84,7 +87,11 @@ func test(t *testing.T, sysTarget *targets.Target) {
 	requests := make(chan *RunRequest, 2*runtime.GOMAXPROCS(0))
 	go func() {
 		for req := range requests {
-			RunTest(req, executor)
+			if req.Bin != "" {
+				runTestC(req)
+			} else {
+				runTest(req, executor)
+			}
 			close(req.Done)
 		}
 	}()
@@ -108,6 +115,63 @@ func test(t *testing.T, sysTarget *targets.Target) {
 	}
 }
 
+func runTest(req *RunRequest, executor string) {
+	cfg := new(ipc.Config)
+	sysTarget := targets.Get(req.P.Target.OS, req.P.Target.Arch)
+	cfg.UseShmem = sysTarget.ExecutorUsesShmem
+	cfg.UseForkServer = sysTarget.ExecutorUsesForkServer
+	cfg.Timeouts = sysTarget.Timeouts(1)
+	cfg.Executor = executor
+	env, err := ipc.MakeEnv(cfg, 0)
+	if err != nil {
+		req.Err = fmt.Errorf("failed to create ipc env: %w", err)
+		return
+	}
+	defer env.Close()
+	for run := 0; run < req.Repeat; run++ {
+		if run%2 == 0 {
+			// Recreate Env every few iterations, this allows to cover more paths.
+			env.ForceRestart()
+		}
+		output, info, hanged, err := env.Exec(&req.Opts, req.P)
+		req.Output = append(req.Output, output...)
+		if err != nil {
+			req.Err = fmt.Errorf("run %v: failed to run: %w", run, err)
+			return
+		}
+		if hanged {
+			req.Err = fmt.Errorf("run %v: hanged", run)
+			return
+		}
+		if run == 0 {
+			req.Info = *info
+		} else {
+			req.Info.Calls = append(req.Info.Calls, info.Calls...)
+		}
+	}
+}
+
+func runTestC(req *RunRequest) {
+	tmpDir, err := os.MkdirTemp("", "syz-runtest")
+	if err != nil {
+		req.Err = fmt.Errorf("failed to create temp dir: %w", err)
+		return
+	}
+	defer os.RemoveAll(tmpDir)
+	cmd := osutil.Command(req.Bin)
+	cmd.Dir = tmpDir
+	// Tell ASAN to not mess with our NONFAILING.
+	cmd.Env = append(append([]string{}, os.Environ()...), "ASAN_OPTIONS=handle_segv=0 allow_user_segv_handler=1")
+	req.Output, req.Err = osutil.Run(20*time.Second, cmd)
+	var verr *osutil.VerboseError
+	if errors.As(req.Err, &verr) {
+		// The process can legitimately do something like exit_group(1).
+		// So we ignore the error and rely on the rest of the checks (e.g. syscall return values).
+		req.Err = nil
+		req.Output = verr.Output
+	}
+}
+
 func TestParsing(t *testing.T) {
 	t.Parallel()
 	// Test only one target in race mode (we have gazillion of auto-generated Linux test).