executor: remove noshmem mode

All OSes we have now support shmem.
Support for Fuchia/Starnix/Windows wasn't implemented,
but generally they support shared memory.
Remove all of the complexity and code associated with noshmem mode.
If/when we revive these OSes, it's easier to properly
implement shmem mode for them.

3 files changed, 24 insertions, 46 deletions

diff --git a/pkg/ipc/ipc.go b/pkg/ipc/ipc.go
index 84a7b9541..2cde21ce4 100644
--- a/pkg/ipc/ipc.go
+++ b/pkg/ipc/ipc.go
@@ -29,7 +29,6 @@ type Config struct {
 	// Path to executor binary.
 	Executor string
 
-	UseShmem      bool // use shared memory instead of pipes for communication
 	UseForkServer bool // use extended protocol with handshake
 	RateLimit     bool // rate limit start of new processes for host fuzzer mode
 
@@ -152,29 +151,25 @@ func MakeEnv(config *Config, pid int) (*Env, error) {
 	}
 	var inf, outf *os.File
 	var inmem, outmem []byte
-	if config.UseShmem {
-		var err error
-		inf, inmem, err = osutil.CreateMemMappedFile(prog.ExecBufferSize)
-		if err != nil {
-			return nil, err
-		}
-		defer func() {
-			if inf != nil {
-				osutil.CloseMemMappedFile(inf, inmem)
-			}
-		}()
-		outf, outmem, err = osutil.CreateMemMappedFile(outputSize)
-		if err != nil {
-			return nil, err
+	var err error
+	inf, inmem, err = osutil.CreateMemMappedFile(prog.ExecBufferSize)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if inf != nil {
+			osutil.CloseMemMappedFile(inf, inmem)
 		}
-		defer func() {
-			if outf != nil {
-				osutil.CloseMemMappedFile(outf, outmem)
-			}
-		}()
-	} else {
-		outmem = make([]byte, outputSize)
+	}()
+	outf, outmem, err = osutil.CreateMemMappedFile(outputSize)
+	if err != nil {
+		return nil, err
 	}
+	defer func() {
+		if outf != nil {
+			osutil.CloseMemMappedFile(outf, outmem)
+		}
+	}()
 	env := &Env{
 		in:      inmem,
 		out:     outmem,
@@ -249,10 +244,7 @@ func (env *Env) ExecProg(opts *flatrpc.ExecOpts, progData []byte) (
 		return
 	}
 	// Copy-in serialized program.
-	if env.config.UseShmem {
-		copy(env.in, progData)
-		progData = nil
-	}
+	copy(env.in, progData)
 	// Zero out the first two words (ncmd and nsig), so that we don't have garbage there
 	// if executor crashes before writing non-garbage there.
 	for i := 0; i < 4; i++ {
@@ -265,7 +257,7 @@ func (env *Env) ExecProg(opts *flatrpc.ExecOpts, progData []byte) (
 	}
 
 	start := osutil.MonotonicNano()
-	output, hanged, err0 = env.cmd.exec(opts, progData)
+	output, hanged, err0 = env.cmd.exec(opts)
 	elapsed := osutil.MonotonicNano() - start
 	if err0 != nil {
 		env.cmd.close()
@@ -521,9 +513,6 @@ type executeReq struct {
 	syscallTimeoutMS uint64
 	programTimeoutMS uint64
 	slowdownScale    uint64
-	progSize         uint64
-	// This structure is followed by a serialized test program in encodingexec format.
-	// Both when sent over a pipe or in shared memory.
 }
 
 type executeReply struct {
@@ -737,7 +726,7 @@ func (c *command) wait() error {
 	return <-c.exited
 }
 
-func (c *command) exec(opts *flatrpc.ExecOpts, progData []byte) (output []byte, hanged bool, err0 error) {
+func (c *command) exec(opts *flatrpc.ExecOpts) (output []byte, hanged bool, err0 error) {
 	if c.flags != opts.EnvFlags || c.sandboxArg != opts.SandboxArg {
 		panic("wrong command")
 	}
@@ -749,7 +738,6 @@ func (c *command) exec(opts *flatrpc.ExecOpts, progData []byte) (output []byte,
 		syscallTimeoutMS: uint64(c.config.Timeouts.Syscall / time.Millisecond),
 		programTimeoutMS: uint64(c.config.Timeouts.Program / time.Millisecond),
 		slowdownScale:    uint64(c.config.Timeouts.Scale),
-		progSize:         uint64(len(progData)),
 	}
 	reqData := (*[unsafe.Sizeof(*req)]byte)(unsafe.Pointer(req))[:]
 	if _, err := c.outwp.Write(reqData); err != nil {
@@ -757,13 +745,6 @@ func (c *command) exec(opts *flatrpc.ExecOpts, progData []byte) (output []byte,
 		err0 = fmt.Errorf("executor %v: failed to write control pipe: %w", c.pid, err)
 		return
 	}
-	if progData != nil {
-		if _, err := c.outwp.Write(progData); err != nil {
-			output = <-c.readDone
-			err0 = fmt.Errorf("executor %v: failed to write control pipe: %w", c.pid, err)
-			return
-		}
-	}
 	// At this point program is executing.
 
 	done := make(chan bool)
diff --git a/pkg/ipc/ipc_test.go b/pkg/ipc/ipc_test.go
index 74a055635..c70bfe79c 100644
--- a/pkg/ipc/ipc_test.go
+++ b/pkg/ipc/ipc_test.go
@@ -23,7 +23,7 @@ import (
 	"github.com/google/syzkaller/sys/targets"
 )
 
-func initTest(t *testing.T) (*prog.Target, rand.Source, int, bool, bool, targets.Timeouts) {
+func initTest(t *testing.T) (*prog.Target, rand.Source, int, bool, targets.Timeouts) {
 	t.Parallel()
 	iters := 100
 	if testing.Short() {
@@ -38,7 +38,7 @@ func initTest(t *testing.T) (*prog.Target, rand.Source, int, bool, bool, targets
 		t.Fatal(err)
 	}
 	rs := testutil.RandSource(t)
-	return target, rs, iters, cfg.UseShmem, cfg.UseForkServer, cfg.Timeouts
+	return target, rs, iters, cfg.UseForkServer, cfg.Timeouts
 }
 
 // TestExecutor runs all internal executor unit tests.
@@ -77,7 +77,7 @@ func prepareTestProgram(target *prog.Target) *prog.Prog {
 }
 
 func TestExecute(t *testing.T) {
-	target, _, _, useShmem, useForkServer, timeouts := initTest(t)
+	target, _, _, useForkServer, timeouts := initTest(t)
 
 	bin := csource.BuildExecutor(t, target, "../..")
 
@@ -86,7 +86,6 @@ func TestExecute(t *testing.T) {
 		t.Logf("testing flags 0x%x", flag)
 		cfg := &Config{
 			Executor:      bin,
-			UseShmem:      useShmem,
 			UseForkServer: useForkServer,
 			Timeouts:      timeouts,
 		}
@@ -122,11 +121,10 @@ func TestExecute(t *testing.T) {
 }
 
 func TestParallel(t *testing.T) {
-	target, _, _, useShmem, useForkServer, timeouts := initTest(t)
+	target, _, _, useForkServer, timeouts := initTest(t)
 	bin := csource.BuildExecutor(t, target, "../..")
 	cfg := &Config{
 		Executor:      bin,
-		UseShmem:      useShmem,
 		UseForkServer: useForkServer,
 		Timeouts:      timeouts,
 	}
diff --git a/pkg/ipc/ipcconfig/ipcconfig.go b/pkg/ipc/ipcconfig/ipcconfig.go
index 3e4b6fd8e..aef709a23 100644
--- a/pkg/ipc/ipcconfig/ipcconfig.go
+++ b/pkg/ipc/ipcconfig/ipcconfig.go
@@ -28,7 +28,6 @@ func Default(target *prog.Target) (*ipc.Config, *flatrpc.ExecOpts, error) {
 		Executor: *flagExecutor,
 		Timeouts: sysTarget.Timeouts(*flagSlowdown),
 	}
-	c.UseShmem = sysTarget.ExecutorUsesShmem
 	c.UseForkServer = sysTarget.ExecutorUsesForkServer
 	c.RateLimit = sysTarget.HostFuzzer && target.OS != targets.TestOS