pkg/ipc: make it possible to change EnvFlags between executions

Pass EnvFlags into Exec instead of New.
This allows to change EnvFlags between executions.
Change of EnvFlags forces executor process restart
since it uses EnvFlags during setup.
Currently this is intended to be NFC since we always
pass the same EnvFlags.
In future this will allow to (1) reduce part of the
VM checking procedure to execution of programs with
different options (e.g. we can probe for coverage/comparisons
support, probe different sandboxes, etc);
(2) use it during fuzzing/reproduction, e.g. we can check
if the crash reproduces under setuid sandbox, or execute some
fuzzing programs in significantly different modes.

4 files changed, 76 insertions, 65 deletions

diff --git a/pkg/ipc/ipc.go b/pkg/ipc/ipc.go
index daebc257d..ebdca1ef8 100644
--- a/pkg/ipc/ipc.go
+++ b/pkg/ipc/ipc.go
@@ -58,7 +58,11 @@ const (
 )
 
 type ExecOpts struct {
-	Flags ExecFlags
+	// Changing ExecFlags between executions does not cause executor process restart.
+	// Changing EnvFlags/SandboxArg does cause process restart.
+	ExecFlags  ExecFlags
+	EnvFlags   EnvFlags
+	SandboxArg int
 }
 
 // Config is the configuration for Env.
@@ -70,10 +74,6 @@ type Config struct {
 	UseForkServer bool // use extended protocol with handshake
 	RateLimit     bool // rate limit start of new processes for host fuzzer mode
 
-	// Flags are configuation flags, defined above.
-	Flags      EnvFlags
-	SandboxArg int
-
 	Timeouts targets.Timeouts
 }
 
@@ -267,7 +267,7 @@ func (env *Env) ExecProg(opts *ExecOpts, progData []byte) (output []byte, info *
 		env.out[i] = 0
 	}
 
-	err0 = env.RestartIfNeeded()
+	err0 = env.RestartIfNeeded(opts)
 	if err0 != nil {
 		return
 	}
@@ -311,9 +311,12 @@ func (env *Env) ForceRestart() {
 }
 
 // RestartIfNeeded brings up an executor process if it was stopped.
-func (env *Env) RestartIfNeeded() error {
+func (env *Env) RestartIfNeeded(opts *ExecOpts) error {
 	if env.cmd != nil {
-		return nil
+		if env.cmd.flags == opts.EnvFlags && env.cmd.sandboxArg == opts.SandboxArg {
+			return nil
+		}
+		env.ForceRestart()
 	}
 	if env.config.RateLimit {
 		rateLimiterOnce.Do(func() {
@@ -321,9 +324,8 @@ func (env *Env) RestartIfNeeded() error {
 		})
 		<-rateLimiter
 	}
-	tmpDirPath := "./"
 	var err error
-	env.cmd, err = makeCommand(env.pid, env.bin, env.config, env.inFile, env.outFile, env.out, tmpDirPath)
+	env.cmd, err = env.makeCommand(opts, "./")
 	return err
 }
 
@@ -381,7 +383,7 @@ func (env *Env) parseOutput(opts *ExecOpts, ncalls int) (*ProgInfo, error) {
 	if len(extraParts) == 0 {
 		return info, nil
 	}
-	info.Extra = convertExtra(extraParts, opts.Flags&FlagDedupCover > 0)
+	info.Extra = convertExtra(extraParts, opts.ExecFlags&FlagDedupCover > 0)
 	return info, nil
 }
 
@@ -491,17 +493,19 @@ func readUint32Array(outp *[]byte, size uint32) ([]uint32, bool) {
 }
 
 type command struct {
-	pid       int
-	config    *Config
-	timeout   time.Duration
-	cmd       *exec.Cmd
-	dir       string
-	readDone  chan []byte
-	exited    chan error
-	inrp      *os.File
-	outwp     *os.File
-	outmem    []byte
-	freshness int
+	pid        int
+	config     *Config
+	flags      EnvFlags
+	sandboxArg int
+	timeout    time.Duration
+	cmd        *exec.Cmd
+	dir        string
+	readDone   chan []byte
+	exited     chan error
+	inrp       *os.File
+	outwp      *os.File
+	outmem     []byte
+	freshness  int
 }
 
 const (
@@ -553,16 +557,15 @@ type callReply struct {
 	// signal/cover/comps follow
 }
 
-func makeCommand(pid int, bin []string, config *Config, inFile, outFile *os.File, outmem []byte,
-	tmpDirPath string) (*command, error) {
-	dir, err := os.MkdirTemp(tmpDirPath, "syzkaller-testdir")
+func (env *Env) makeCommand(opts *ExecOpts, tmpDir string) (*command, error) {
+	dir, err := os.MkdirTemp(tmpDir, "syzkaller-testdir")
 	if err != nil {
 		return nil, fmt.Errorf("failed to create temp dir: %w", err)
 	}
 	dir = osutil.Abs(dir)
 
-	timeout := config.Timeouts.Program
-	if config.UseForkServer {
+	timeout := env.config.Timeouts.Program
+	if env.config.UseForkServer {
 		// Executor has an internal timeout and protects against most hangs when fork server is enabled,
 		// so we use quite large timeout. Executor can be slow due to global locks in namespaces
 		// and other things, so let's better wait than report false misleading crashes.
@@ -570,11 +573,13 @@ func makeCommand(pid int, bin []string, config *Config, inFile, outFile *os.File
 	}
 
 	c := &command{
-		pid:     pid,
-		config:  config,
-		timeout: timeout,
-		dir:     dir,
-		outmem:  outmem,
+		pid:        env.pid,
+		config:     env.config,
+		flags:      opts.EnvFlags,
+		sandboxArg: opts.SandboxArg,
+		timeout:    timeout,
+		dir:        dir,
+		outmem:     env.out,
 	}
 	defer func() {
 		if c != nil {
@@ -611,16 +616,16 @@ func makeCommand(pid int, bin []string, config *Config, inFile, outFile *os.File
 
 	c.readDone = make(chan []byte, 1)
 
-	cmd := osutil.Command(bin[0], bin[1:]...)
-	if inFile != nil && outFile != nil {
-		cmd.ExtraFiles = []*os.File{inFile, outFile}
+	cmd := osutil.Command(env.bin[0], env.bin[1:]...)
+	if env.inFile != nil && env.outFile != nil {
+		cmd.ExtraFiles = []*os.File{env.inFile, env.outFile}
 	}
 	cmd.Dir = dir
 	// Tell ASAN to not mess with our NONFAILING.
 	cmd.Env = append(append([]string{}, os.Environ()...), "ASAN_OPTIONS=handle_segv=0 allow_user_segv_handler=1")
 	cmd.Stdin = outrp
 	cmd.Stdout = inwp
-	if config.Flags&FlagDebug != 0 {
+	if c.flags&FlagDebug != 0 {
 		close(c.readDone)
 		cmd.Stderr = os.Stdout
 	} else {
@@ -694,9 +699,9 @@ func (c *command) close() {
 func (c *command) handshake() error {
 	req := &handshakeReq{
 		magic:      inMagic,
-		flags:      uint64(c.config.Flags),
+		flags:      uint64(c.flags),
 		pid:        uint64(c.pid),
-		sandboxArg: uint64(c.config.SandboxArg),
+		sandboxArg: uint64(c.sandboxArg),
 	}
 	reqData := (*[unsafe.Sizeof(*req)]byte)(unsafe.Pointer(req))[:]
 	if _, err := c.outwp.Write(reqData); err != nil {
@@ -744,10 +749,13 @@ func (c *command) wait() error {
 }
 
 func (c *command) exec(opts *ExecOpts, progData []byte) (output []byte, hanged bool, err0 error) {
+	if c.flags != opts.EnvFlags || c.sandboxArg != opts.SandboxArg {
+		panic("wrong command")
+	}
 	req := &executeReq{
 		magic:            inMagic,
-		envFlags:         uint64(c.config.Flags),
-		execFlags:        uint64(opts.Flags),
+		envFlags:         uint64(c.flags),
+		execFlags:        uint64(opts.ExecFlags),
 		pid:              uint64(c.pid),
 		syscallTimeoutMS: uint64(c.config.Timeouts.Syscall / time.Millisecond),
 		programTimeoutMS: uint64(c.config.Timeouts.Program / time.Millisecond),
diff --git a/pkg/ipc/ipc_priv_test.go b/pkg/ipc/ipc_priv_test.go
index 054916045..94c48af75 100644
--- a/pkg/ipc/ipc_priv_test.go
+++ b/pkg/ipc/ipc_priv_test.go
@@ -9,11 +9,16 @@ import (
 
 func TestOutputDeadline(t *testing.T) {
 	// Run the command that leaks stderr to a child process.
-	c, err := makeCommand(1, []string{
-		"sh",
-		"-c",
-		"exec 1>&2; ( sleep 100; echo fail ) & echo done",
-	}, &Config{}, nil, nil, nil, "/tmp")
+	env := &Env{
+		bin: []string{
+			"sh",
+			"-c",
+			"exec 1>&2; ( sleep 100; echo fail ) & echo done",
+		},
+		pid:    1,
+		config: &Config{},
+	}
+	c, err := env.makeCommand(&ExecOpts{}, t.TempDir())
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/pkg/ipc/ipc_test.go b/pkg/ipc/ipc_test.go
index ffb28d0a6..fcc8b5dcd 100644
--- a/pkg/ipc/ipc_test.go
+++ b/pkg/ipc/ipc_test.go
@@ -100,7 +100,6 @@ func TestExecute(t *testing.T) {
 			UseShmem:      useShmem,
 			UseForkServer: useForkServer,
 			Timeouts:      timeouts,
-			SandboxArg:    0,
 		}
 		env, err := MakeEnv(cfg, 0)
 		if err != nil {
@@ -111,7 +110,7 @@ func TestExecute(t *testing.T) {
 		for i := 0; i < 10; i++ {
 			p := prepareTestProgram(target)
 			opts := &ExecOpts{
-				Flags: flag,
+				ExecFlags: flag,
 			}
 			output, info, hanged, err := env.Exec(opts, p)
 			if err != nil {
@@ -142,7 +141,6 @@ func TestParallel(t *testing.T) {
 		UseShmem:      useShmem,
 		UseForkServer: useForkServer,
 		Timeouts:      timeouts,
-		SandboxArg:    0,
 	}
 	const P = 10
 	errs := make(chan error, P)
@@ -204,7 +202,7 @@ func TestZlib(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	cfg.Flags |= FlagDebug
+	opts.EnvFlags |= FlagDebug
 	cfg.Executor = buildExecutor(t, target)
 	defer os.Remove(cfg.Executor)
 	env, err := MakeEnv(cfg, 0)
diff --git a/pkg/ipc/ipcconfig/ipcconfig.go b/pkg/ipc/ipcconfig/ipcconfig.go
index 4b4aacd8a..1a91b28fb 100644
--- a/pkg/ipc/ipcconfig/ipcconfig.go
+++ b/pkg/ipc/ipcconfig/ipcconfig.go
@@ -27,30 +27,30 @@ func Default(target *prog.Target) (*ipc.Config, *ipc.ExecOpts, error) {
 		Executor: *flagExecutor,
 		Timeouts: sysTarget.Timeouts(*flagSlowdown),
 	}
-	if *flagSignal {
-		c.Flags |= ipc.FlagSignal
-	}
-	if *flagDebug {
-		c.Flags |= ipc.FlagDebug
-	}
-	sandboxFlags, err := ipc.SandboxToFlags(*flagSandbox)
-	if err != nil {
-		return nil, nil, err
-	}
-	c.SandboxArg = *flagSandboxArg
-	c.Flags |= sandboxFlags
 	c.UseShmem = sysTarget.ExecutorUsesShmem
 	c.UseForkServer = sysTarget.ExecutorUsesForkServer
 	c.RateLimit = sysTarget.HostFuzzer && target.OS != targets.TestOS
+
 	opts := &ipc.ExecOpts{
-		Flags: ipc.FlagDedupCover,
+		ExecFlags: ipc.FlagDedupCover,
 	}
 	if *flagThreaded {
-		opts.Flags |= ipc.FlagThreaded
+		opts.ExecFlags |= ipc.FlagThreaded
 	}
 	if *flagSignal {
-		opts.Flags |= ipc.FlagCollectSignal
+		opts.ExecFlags |= ipc.FlagCollectSignal
 	}
-
+	if *flagSignal {
+		opts.EnvFlags |= ipc.FlagSignal
+	}
+	if *flagDebug {
+		opts.EnvFlags |= ipc.FlagDebug
+	}
+	sandboxFlags, err := ipc.SandboxToFlags(*flagSandbox)
+	if err != nil {
+		return nil, nil, err
+	}
+	opts.SandboxArg = *flagSandboxArg
+	opts.EnvFlags |= sandboxFlags
 	return c, opts, nil
 }