use fork server in executor

This avoids exec per test.
Also allows to pre-map shared memory regions.
And will allow to pre-map coverage regions, etc.

Seems to work already, but probably there are still some bugs.

4 files changed, 353 insertions, 62 deletions

diff --git a/executor/executor.cc b/executor/executor.cc
index e126d466d..2c2aa9841 100644
--- a/executor/executor.cc
+++ b/executor/executor.cc
@@ -11,11 +11,14 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <time.h>
+#include <signal.h>
+#include <limits.h>
 #include <sys/types.h>
 #include <sys/time.h>
 #include <sys/syscall.h>
 #include <sys/stat.h>
 #include <sys/mman.h>
+#include <sys/wait.h>
 #include <linux/futex.h>
 #include <pthread.h>
 #include <grp.h>
@@ -24,6 +27,8 @@
 
 const int kInFd = 3;
 const int kOutFd = 4;
+const int kInPipeFd = 5;
+const int kOutPipeFd = 6;
 const int kCoverFd = 5;
 const int kMaxInput = 1 << 20;
 const int kMaxOutput = 16 << 20;
@@ -39,6 +44,9 @@ const uint64_t arg_const = 0;
 const uint64_t arg_result = 1;
 const uint64_t arg_data = 2;
 
+const int kFailStatus = 67;
+const int kErrorStatus = 68;
+
 // We use the default value instead of results of failed syscalls.
 // -1 is an invalid fd and an invalid address and deterministic,
 // so good enough for our purposes.
@@ -89,6 +97,7 @@ __attribute__((noreturn)) void fail(const char* msg, ...);
 __attribute__((noreturn)) void error(const char* msg, ...);
 __attribute__((noreturn)) void exitf(const char* msg, ...);
 void debug(const char* msg, ...);
+void execute_one();
 uint64_t read_input(uint64_t** input_posp, bool peek = false);
 uint64_t read_arg(uint64_t** input_posp);
 uint64_t read_result(uint64_t** input_posp);
@@ -112,35 +121,84 @@ int main()
 		fail("mmap of input file failed");
 	if (mmap(&output_data[0], kMaxOutput, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, kOutFd, 0) != &output_data[0])
 		fail("mmap of output file failed");
-retry:
-	uint64_t* input_pos = (uint64_t*)&input_data[0];
-	uint64_t flags = read_input(&input_pos);
+	char cwdbuf[64 << 10];
+	char* cwd = getcwd(cwdbuf, sizeof(cwdbuf));
+
+	sigset_t sigchldset;
+	sigemptyset(&sigchldset);
+	sigaddset(&sigchldset, SIGCHLD);
+	if (sigprocmask(SIG_BLOCK, &sigchldset, NULL))
+		fail("sigprocmask failed");
+
+	uint64_t flags = *(uint64_t*)input_data;
 	flag_debug = flags & (1 << 0);
 	flag_cover = flags & (1 << 1);
 	flag_threaded = flags & (1 << 2);
 	flag_collide = flags & (1 << 3);
 	flag_deduplicate = flags & (1 << 4);
 	flag_drop_privs = flags & (1 << 5);
-	output_pos = (uint32_t*)&output_data[0];
-	write_output(0); // Number of executed syscalls (updated later).
-
 	if (flag_collide)
 		flag_threaded = true;
-	if (!collide) {
-		cover_open();
-		if (!flag_threaded)
-			cover_init(&threads[0]);
-
-		if (flag_drop_privs) {
-			// TODO: 65534 is meant to be nobody
-			if (setgroups(0, NULL))
-				fail("failed to setgroups");
-			if (setresgid(65534, 65534, 65534))
-				fail("failed to setresgid");
-			if (setresuid(65534, 65534, 65534))
-				fail("failed to setresuid");
+
+	cover_open();
+
+	for (;;) {
+		char tmp;
+		if (read(kInPipeFd, &tmp, 1) != 1)
+			fail("control pipe read failed");
+		// The dir may have been recreated.
+		if (chdir(cwd))
+			fail("failed to chdir");
+
+		int pid = fork();
+		if (pid < 0)
+			fail("fork failed");
+		if (pid == 0) {
+			setpgid(0, 0);
+			if (flag_drop_privs) {
+				// TODO: 65534 is meant to be nobody
+				if (setgroups(0, NULL))
+					fail("failed to setgroups");
+				if (setresgid(65534, 65534, 65534))
+					fail("failed to setresgid");
+				if (setresuid(65534, 65534, 65534))
+					fail("failed to setresuid");
+			}
+			execute_one();
+			debug("exiting\n");
+			return 0;
 		}
+
+		timespec ts = {};
+		ts.tv_sec = 5;
+		ts.tv_nsec = 0;
+		if (sigtimedwait(&sigchldset, NULL, &ts) < 0) {
+			kill(-pid, SIGKILL);
+			kill(pid, SIGKILL);
+		}
+		int status = 0;
+		if (waitpid(pid, &status, 0) != pid)
+			fail("waitpid failed");
+		status = WEXITSTATUS(status);
+		if (status == kFailStatus)
+			fail("child failed");
+		if (status == kErrorStatus)
+			error("child errored");
+		if (write(kOutPipeFd, &tmp, 1) != 1)
+			fail("control pipe write failed");
 	}
+}
+
+void execute_one()
+{
+retry:
+	uint64_t* input_pos = (uint64_t*)&input_data[0];
+	read_input(&input_pos); // flags
+	output_pos = (uint32_t*)&output_data[0];
+	write_output(0); // Number of executed syscalls (updated later).
+
+	if (!collide && !flag_threaded)
+		cover_init(&threads[0]);
 
 	int call_index = 0;
 	for (int n = 0;; n++) {
@@ -236,16 +294,11 @@ retry:
 		}
 	}
 
-	if (flag_collide) {
-		if (!collide) {
-			debug("enabling collider\n");
-			collide = true;
-			goto retry;
-		}
+	if (flag_collide && !collide) {
+		debug("enabling collider\n");
+		collide = true;
+		goto retry;
 	}
-
-	debug("exiting\n");
-	return 0;
 }
 
 thread_t* schedule_call(int n, int call_index, int call_num, uint64_t num_args, uint64_t* args, uint64_t* pos)
@@ -537,7 +590,7 @@ void fail(const char* msg, ...)
 	vfprintf(stderr, msg, args);
 	va_end(args);
 	fprintf(stderr, " (errno %d)\n", e);
-	exit(67);
+	exit(kFailStatus);
 }
 
 // kernel error (e.g. wrong syscall return value)
@@ -549,7 +602,7 @@ void error(const char* msg, ...)
 	vfprintf(stderr, msg, args);
 	va_end(args);
 	fprintf(stderr, "\n");
-	exit(68);
+	exit(kErrorStatus);
 }
 
 // just exit (e.g. due to temporal ENOMEM error)
diff --git a/fuzzer/fuzzer.go b/fuzzer/fuzzer.go
index cb78d848d..cdcbcfca6 100644
--- a/fuzzer/fuzzer.go
+++ b/fuzzer/fuzzer.go
@@ -109,7 +109,7 @@ func main() {
 	if !*flagNoCover {
 		flags |= ipc.FlagCover | ipc.FlagDedupCover
 	}
-	env, err := ipc.MakeEnv(*flagExecutor, 4*time.Second, flags)
+	env, err := ipc.MakeEnv(*flagExecutor, 10*time.Second, flags)
 	if err != nil {
 		panic(err)
 	}
diff --git a/ipc/ipc.go b/ipc/ipc.go
index 4c881228b..7673179a7 100644
--- a/ipc/ipc.go
+++ b/ipc/ipc.go
@@ -22,6 +22,7 @@ type Env struct {
 	In  []byte
 	Out []byte
 
+	cmd     *command
 	inFile  *os.File
 	outFile *os.File
 	bin     []string
@@ -74,12 +75,19 @@ func MakeEnv(bin string, timeout time.Duration, flags uint64) (*Env, error) {
 	if len(env.bin) == 0 {
 		return nil, fmt.Errorf("binary is empty string")
 	}
+	env.bin[0], err = filepath.Abs(env.bin[0]) // we are going to chdir
+	if err != nil {
+		return nil, fmt.Errorf("filepath.Abs failed: %v", err)
+	}
 	inf = nil
 	outf = nil
 	return env, nil
 }
 
 func (env *Env) Close() error {
+	if env.cmd != nil {
+		env.cmd.close()
+	}
 	err1 := closeMapping(env.inFile, env.In)
 	err2 := closeMapping(env.outFile, env.Out)
 	switch {
@@ -116,8 +124,16 @@ func (env *Env) Exec(p *prog.Prog) (output, strace []byte, cov [][]uint32, faile
 		}
 	}
 
-	output, strace, failed, hanged, err0 = env.execBin()
+	if env.cmd == nil {
+		env.cmd, err0 = makeCommand(env.bin, env.timeout, env.flags, env.inFile, env.outFile)
+		if err0 != nil {
+			return
+		}
+	}
+	output, strace, failed, hanged, err0 = env.cmd.exec()
 	if err0 != nil {
+		env.cmd.close()
+		env.cmd = nil
 		return
 	}
 
@@ -172,12 +188,15 @@ func (env *Env) Exec(p *prog.Prog) (output, strace []byte, cov [][]uint32, faile
 	return
 }
 
+/*
 func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 error) {
 	dir, err := ioutil.TempDir("./", "syzkaller-testdir")
 	if err != nil {
 		err0 = fmt.Errorf("failed to create temp dir: %v", err)
 		return
 	}
+
+	// Output capture pipe.
 	defer os.RemoveAll(dir)
 	rp, wp, err := os.Pipe()
 	if err != nil {
@@ -186,6 +205,25 @@ func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 erro
 	}
 	defer rp.Close()
 	defer wp.Close()
+
+	// Input command pipe.
+	inrp, inwp, err := os.Pipe()
+	if err != nil {
+		err0 = fmt.Errorf("failed to create pipe: %v", err)
+		return
+	}
+	defer inrp.Close()
+	defer inwp.Close()
+
+	// Output command pipe.
+	outrp, outwp, err := os.Pipe()
+	if err != nil {
+		err0 = fmt.Errorf("failed to create pipe: %v", err)
+		return
+	}
+	defer outrp.Close()
+	defer outwp.Close()
+
 	cmd := exec.Command(env.bin[0], env.bin[1:]...)
 	traceFile := ""
 	if env.flags&FlagStrace != 0 {
@@ -204,7 +242,7 @@ func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 erro
 		}
 		cmd = exec.Command("strace", args...)
 	}
-	cmd.ExtraFiles = append(cmd.ExtraFiles, env.inFile, env.outFile)
+	cmd.ExtraFiles = append(cmd.ExtraFiles, env.inFile, env.outFile, outrp, inwp)
 	cmd.Env = []string{}
 	cmd.Dir = dir
 	if env.flags&FlagDebug == 0 {
@@ -220,11 +258,17 @@ func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 erro
 	} else {
 		cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
 	}
+	if _, err := outwp.Write([]byte{0}); err != nil {
+		err0 = fmt.Errorf("failed to write control pipe: %v", err)
+		return
+	}
 	if err := cmd.Start(); err != nil {
 		err0 = fmt.Errorf("failed to start executor binary: %v", err)
 		return
 	}
 	wp.Close()
+	outrp.Close()
+	inwp.Close()
 	done := make(chan bool)
 	hang := make(chan bool)
 	go func() {
@@ -234,6 +278,7 @@ func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 erro
 			// We started the process in its own process group and now kill the whole group.
 			// This solves a potential problem with strace:
 			// if we kill just strace, executor still runs and ReadAll below hangs.
+			fmt.Printf("KILLING %v\n", cmd.Process.Pid)
 			syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
 			syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
 			syscall.Kill(cmd.Process.Pid, syscall.SIGKILL)
@@ -244,6 +289,11 @@ func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 erro
 			hang <- false
 		}
 	}()
+	var tmp [1]byte
+	if n, err := inrp.Read(tmp[:]); n != 1 || err != nil {
+		err0 = fmt.Errorf("failed to read control pipe: %v", err)
+		return
+	}
 	output, err = ioutil.ReadAll(rp)
 	readErr := err
 	close(done)
@@ -281,6 +331,7 @@ func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 erro
 	}
 	return
 }
+*/
 
 func createMapping(size int) (f *os.File, mem []byte, err error) {
 	f, err = ioutil.TempFile("./", "syzkaller-shm")
@@ -326,3 +377,187 @@ func closeMapping(f *os.File, mem []byte) error {
 		return nil
 	}
 }
+
+type command struct {
+	timeout time.Duration
+	cmd     *exec.Cmd
+	dir     string
+	rp      *os.File
+	inrp    *os.File
+	outwp   *os.File
+}
+
+func makeCommand(bin []string, timeout time.Duration, flags uint64, inFile *os.File, outFile *os.File) (*command, error) {
+	dir, err := ioutil.TempDir("./", "syzkaller-testdir")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temp dir: %v", err)
+	}
+	if err := os.Chmod(dir, 0777); err != nil {
+		return nil, fmt.Errorf("failed to chmod temp dir: %v", err)
+	}
+
+	c := &command{timeout: timeout, dir: dir}
+	defer func() {
+		if c != nil {
+			c.close()
+		}
+	}()
+
+	// Output capture pipe.
+	rp, wp, err := os.Pipe()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create pipe: %v", err)
+	}
+	defer wp.Close()
+	c.rp = rp
+
+	// Input command pipe.
+	inrp, inwp, err := os.Pipe()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create pipe: %v", err)
+	}
+	defer inwp.Close()
+	c.inrp = inrp
+
+	// Output command pipe.
+	outrp, outwp, err := os.Pipe()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create pipe: %v", err)
+	}
+	defer outrp.Close()
+	c.outwp = outwp
+
+	cmd := exec.Command(bin[0], bin[1:]...)
+	/*
+		traceFile := ""
+		if flags&FlagStrace != 0 {
+			f, err := ioutil.TempFile("./", "syzkaller-strace")
+			if err != nil {
+				return nil, fmt.Errorf("failed to create temp file: %v", err)
+			}
+			f.Close()
+			defer os.Remove(f.Name())
+			traceFile, _ = filepath.Abs(f.Name())
+			args := []string{"-s", "8", "-o", traceFile}
+			args = append(args, env.bin...)
+			if env.flags&FlagThreaded != 0 {
+				args = append([]string{"-f"}, args...)
+			}
+			cmd = exec.Command("strace", args...)
+		}
+	*/
+	cmd.ExtraFiles = []*os.File{inFile, outFile, outrp, inwp}
+	cmd.Env = []string{}
+	cmd.Dir = dir
+	if flags&FlagDebug == 0 {
+		cmd.Stdout = wp
+		cmd.Stderr = wp
+	} else {
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stdout
+	}
+	if syscall.Getuid() == 0 {
+		// Running under root, more isolation is possible.
+		cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Cloneflags: syscall.CLONE_NEWNS}
+	} else {
+		cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	}
+	if err := cmd.Start(); err != nil {
+		return nil, fmt.Errorf("failed to start executor binary: %v", err)
+	}
+	c.cmd = cmd
+	tmp := c
+	c = nil // disable defer above
+	return tmp, nil
+}
+
+func (c *command) close() {
+	c.kill()
+	c.cmd.Wait()
+	os.RemoveAll(c.dir)
+	if c.rp != nil {
+		c.rp.Close()
+	}
+	if c.inrp != nil {
+		c.inrp.Close()
+	}
+	if c.outwp != nil {
+		c.outwp.Close()
+	}
+}
+
+func (c *command) kill() {
+	// We started the process in its own process group and now kill the whole group.
+	// This solves a potential problem with strace:
+	// if we kill just strace, executor still runs and ReadAll below hangs.
+	syscall.Kill(-c.cmd.Process.Pid, syscall.SIGKILL)
+	syscall.Kill(c.cmd.Process.Pid, syscall.SIGKILL)
+}
+
+func (c *command) exec() (output, strace []byte, failed, hanged bool, err0 error) {
+	var tmp [1]byte
+	if _, err := c.outwp.Write(tmp[:]); err != nil {
+		err0 = fmt.Errorf("failed to write control pipe: %v", err)
+		return
+	}
+	done := make(chan bool)
+	hang := make(chan bool)
+	go func() {
+		t := time.NewTimer(c.timeout)
+		select {
+		case <-t.C:
+			c.kill()
+			hang <- true
+		case <-done:
+			t.Stop()
+			hang <- false
+		}
+	}()
+	//!!! handle c.rp overflow
+	_, readErr := c.inrp.Read(tmp[:])
+	close(done)
+	os.RemoveAll(c.dir)
+	if err := os.Mkdir(c.dir, 0777); err != nil {
+		<-hang
+		err0 = fmt.Errorf("failed to create temp dir: %v", err)
+		return
+	}
+	if readErr == nil {
+		<-hang
+		return
+	}
+	err0 = fmt.Errorf("executor did not answer")
+	c.kill()
+	var err error
+	output, err = ioutil.ReadAll(c.rp)
+	if err = c.cmd.Wait(); <-hang && err != nil {
+		hanged = true
+	}
+	if err != nil {
+		output = append(output, []byte(err.Error())...)
+		output = append(output, '\n')
+	}
+	if c.cmd.ProcessState != nil {
+		sys := c.cmd.ProcessState.Sys()
+		if ws, ok := sys.(syscall.WaitStatus); ok {
+			// Magic values returned by executor.
+			if ws.ExitStatus() == 67 {
+				err0 = fmt.Errorf("executor failed: %s", output)
+				return
+			}
+			if ws.ExitStatus() == 68 {
+				failed = true
+			}
+		}
+	}
+	/*
+		if traceFile != "" {
+			strace, err = ioutil.ReadFile(traceFile)
+			if err != nil {
+				err0 = fmt.Errorf("failed to read strace output: %v", err)
+				return
+			}
+		}
+	*/
+	return
+}
diff --git a/tools/execprog/execprog.go b/tools/execprog/execprog.go
index 471808b6c..f85925125 100644
--- a/tools/execprog/execprog.go
+++ b/tools/execprog/execprog.go
@@ -24,17 +24,18 @@ import (
 )
 
 var (
-	flagExecutor = flag.String("executor", "", "path to executor binary")
-	flagThreaded = flag.Bool("threaded", true, "use threaded mode in executor")
-	flagCollide  = flag.Bool("collide", true, "collide syscalls to provoke data races")
-	flagDebug    = flag.Bool("debug", false, "debug output from executor")
-	flagStrace   = flag.Bool("strace", false, "run executor under strace")
-	flagCover    = flag.String("cover", "", "collect coverage and write to the file")
-	flagNobody   = flag.Bool("nobody", true, "impersonate into nobody")
-	flagDedup    = flag.Bool("dedup", false, "deduplicate coverage in executor")
-	flagLoop     = flag.Bool("loop", false, "execute programs in a loop")
-	flagProcs    = flag.Int("procs", 1, "number of parallel processes to execute programs")
-	flagTimeout  = flag.Duration("timeout", 5*time.Second, "execution timeout")
+	flagExecutor  = flag.String("executor", "", "path to executor binary")
+	flagThreaded  = flag.Bool("threaded", true, "use threaded mode in executor")
+	flagCollide   = flag.Bool("collide", true, "collide syscalls to provoke data races")
+	flagDebug     = flag.Bool("debug", false, "debug output from executor")
+	flagStrace    = flag.Bool("strace", false, "run executor under strace")
+	flagCover     = flag.Bool("cover", true, "collect coverage")
+	flagCoverFile = flag.String("coverfile", "", "write coverage to the file")
+	flagNobody    = flag.Bool("nobody", true, "impersonate into nobody")
+	flagDedup     = flag.Bool("dedup", false, "deduplicate coverage in executor")
+	flagLoop      = flag.Bool("loop", false, "execute programs in a loop")
+	flagProcs     = flag.Int("procs", 1, "number of parallel processes to execute programs")
+	flagTimeout   = flag.Duration("timeout", 10*time.Second, "execution timeout")
 )
 
 func main() {
@@ -67,7 +68,7 @@ func main() {
 	if *flagStrace {
 		flags |= ipc.FlagStrace
 	}
-	if *flagCover != "" {
+	if *flagCover || *flagCoverFile != "" {
 		flags |= ipc.FlagCover
 	}
 	if *flagDedup {
@@ -98,28 +99,30 @@ func main() {
 				}
 				p := progs[idx%len(progs)]
 				output, strace, cov, failed, hanged, err := env.Exec(p)
-				if *flagDebug {
+				if *flagDebug || err != nil {
 					fmt.Printf("result: failed=%v hanged=%v err=%v\n\n%s", failed, hanged, err, output)
 				}
 				if *flagStrace {
 					fmt.Printf("strace output:\n%s", strace)
 				}
-				// Coverage is dumped in sanitizer format.
-				// github.com/google/sanitizers/tools/sancov command can be used to dump PCs,
-				// then they can be piped via addr2line to symbolize.
-				for i, c := range cov {
-					fmt.Printf("call #%v: coverage %v\n", i, len(c))
-					if len(c) == 0 {
-						continue
-					}
-					buf := new(bytes.Buffer)
-					binary.Write(buf, binary.LittleEndian, uint64(0xC0BFFFFFFFFFFF64))
-					for _, pc := range c {
-						binary.Write(buf, binary.LittleEndian, cover.RestorePC(pc))
-					}
-					err := ioutil.WriteFile(fmt.Sprintf("%v.%v", *flagCover, i), buf.Bytes(), 0660)
-					if err != nil {
-						log.Fatalf("failed to write coverage file: %v", err)
+				if *flagCoverFile != "" {
+					// Coverage is dumped in sanitizer format.
+					// github.com/google/sanitizers/tools/sancov command can be used to dump PCs,
+					// then they can be piped via addr2line to symbolize.
+					for i, c := range cov {
+						fmt.Printf("call #%v: coverage %v\n", i, len(c))
+						if len(c) == 0 {
+							continue
+						}
+						buf := new(bytes.Buffer)
+						binary.Write(buf, binary.LittleEndian, uint64(0xC0BFFFFFFFFFFF64))
+						for _, pc := range c {
+							binary.Write(buf, binary.LittleEndian, cover.RestorePC(pc))
+						}
+						err := ioutil.WriteFile(fmt.Sprintf("%v.%v", *flagCoverFile, i), buf.Bytes(), 0660)
+						if err != nil {
+							log.Fatalf("failed to write coverage file: %v", err)
+						}
 					}
 				}
 			}