ipc: add an optional 'abort' signal

If an external sandbox process wraps the executor, it may be helpful to
send a signal other than SIGKILL to the sandbox when the program times
out or fails to respond. This gives the sandbox the opportunity to emit
additional debugging information before exiting.

Add an 'abort' signal to ipc, which is sent to the executor before
SIGKILL. If the executor fails to exit within 5s, the signal is upgraded
to SIGKILL.

The default abort signal remains SIGKILL, maintaining existing behavior.

5 files changed, 95 insertions, 53 deletions

diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 5da256247..c5191b9fd 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -9,6 +9,7 @@ Google Inc.
  David Drysdale
  Vishwath Mohan
  Billy Lau
+ Michael Pratt
 Baozeng Ding
 Lorenzo Stoakes
 Jeremy Huang
diff --git a/ipc/ipc.go b/ipc/ipc.go
index 29feb8b85..243b74aa9 100644
--- a/ipc/ipc.go
+++ b/ipc/ipc.go
@@ -29,14 +29,14 @@ type Env struct {
 	inFile  *os.File
 	outFile *os.File
 	bin     []string
-	timeout time.Duration
-	flags   uint64
 	pid     int
+	config  Config
 
 	StatExecs    uint64
 	StatRestarts uint64
 }
 
+// Configuration flags for Config.Flags.
 const (
 	FlagDebug            = uint64(1) << iota // debug output from executor
 	FlagSignal                               // collect feedback signals (coverage)
@@ -45,7 +45,9 @@ const (
 	FlagSandboxSetuid                        // impersonate nobody user
 	FlagSandboxNamespace                     // use namespaces for sandboxing
 	FlagEnableTun                            // initialize and use tun in executor
+)
 
+const (
 	outputSize   = 16 << 20
 	signalOffset = 15 << 20
 
@@ -63,7 +65,8 @@ var (
 	// Executor protects against most hangs, so we use quite large timeout here.
 	// Executor can be slow due to global locks in namespaces and other things,
 	// so let's better wait than report false misleading crashes.
-	flagTimeout = flag.Duration("timeout", 1*time.Minute, "execution timeout")
+	flagTimeout     = flag.Duration("timeout", 1*time.Minute, "execution timeout")
+	flagAbortSignal = flag.Int("abort_signal", int(syscall.SIGKILL), "initial signal to send to executor in error conditions; upgrades to SIGKILL if executor does not exit")
 )
 
 // ExecutorFailure is returned from MakeEnv or from env.Exec when executor terminates by calling fail function.
@@ -74,37 +77,52 @@ func (err ExecutorFailure) Error() string {
 	return string(err)
 }
 
-func DefaultFlags() (uint64, time.Duration, error) {
-	var flags uint64
+// Config is the configuration for Env.
+type Config struct {
+	// Flags are configuation flags, defined above.
+	Flags uint64
+
+	// Timeout is the execution timeout for a single program.
+	Timeout time.Duration
+
+	// AbortSignal is the signal to send to the executor in error
+	// conditions.
+	AbortSignal syscall.Signal
+}
+
+func DefaultConfig() (Config, error) {
+	var c Config
 	if *flagThreaded {
-		flags |= FlagThreaded
+		c.Flags |= FlagThreaded
 	}
 	if *flagCollide {
-		flags |= FlagCollide
+		c.Flags |= FlagCollide
 	}
 	if *flagSignal {
-		flags |= FlagSignal
+		c.Flags |= FlagSignal
 	}
 	switch *flagSandbox {
 	case "none":
 	case "setuid":
-		flags |= FlagSandboxSetuid
+		c.Flags |= FlagSandboxSetuid
 	case "namespace":
-		flags |= FlagSandboxNamespace
+		c.Flags |= FlagSandboxNamespace
 	default:
-		return 0, 0, fmt.Errorf("flag sandbox must contain one of none/setuid/namespace")
+		return Config{}, fmt.Errorf("flag sandbox must contain one of none/setuid/namespace")
 	}
 	if *flagDebug {
-		flags |= FlagDebug
+		c.Flags |= FlagDebug
 	}
-	return flags, *flagTimeout, nil
+	c.Timeout = *flagTimeout
+	c.AbortSignal = syscall.Signal(*flagAbortSignal)
+	return c, nil
 }
 
-func MakeEnv(bin string, timeout time.Duration, flags uint64, pid int) (*Env, error) {
+func MakeEnv(bin string, pid int, config Config) (*Env, error) {
 	// IPC timeout must be larger then executor timeout.
 	// Otherwise IPC will kill parent executor but leave child executor alive.
-	if timeout < 7*time.Second {
-		timeout = 7 * time.Second
+	if config.Timeout < 7*time.Second {
+		config.Timeout = 7 * time.Second
 	}
 	inf, inmem, err := createMapping(prog.ExecBufferSize)
 	if err != nil {
@@ -125,7 +143,7 @@ func MakeEnv(bin string, timeout time.Duration, flags uint64, pid int) (*Env, er
 		}
 	}()
 	for i := 0; i < 8; i++ {
-		inmem[i] = byte(flags >> (8 * uint(i)))
+		inmem[i] = byte(config.Flags >> (8 * uint(i)))
 	}
 	*(*uint64)(unsafe.Pointer(&inmem[8])) = uint64(pid)
 	inmem = inmem[16:]
@@ -135,9 +153,8 @@ func MakeEnv(bin string, timeout time.Duration, flags uint64, pid int) (*Env, er
 		inFile:  inf,
 		outFile: outf,
 		bin:     strings.Split(bin, " "),
-		timeout: timeout,
-		flags:   flags,
 		pid:     pid,
+		config:  config,
 	}
 	if len(env.bin) == 0 {
 		return nil, fmt.Errorf("binary is empty string")
@@ -203,7 +220,7 @@ func (env *Env) Exec(p *prog.Prog, cover, dedup bool) (output []byte, info []Cal
 			return
 		}
 	}
-	if env.flags&FlagSignal != 0 {
+	if env.config.Flags&FlagSignal != 0 {
 		// Zero out the first two words (ncmd and nsig), so that we don't have garbage there
 		// if executor crashes before writing non-garbage there.
 		for i := 0; i < 4; i++ {
@@ -214,7 +231,7 @@ func (env *Env) Exec(p *prog.Prog, cover, dedup bool) (output []byte, info []Cal
 	atomic.AddUint64(&env.StatExecs, 1)
 	if env.cmd == nil {
 		atomic.AddUint64(&env.StatRestarts, 1)
-		env.cmd, err0 = makeCommand(env.pid, env.bin, env.timeout, env.flags, env.inFile, env.outFile)
+		env.cmd, err0 = makeCommand(env.pid, env.bin, env.config, env.inFile, env.outFile)
 		if err0 != nil {
 			return
 		}
@@ -227,7 +244,7 @@ func (env *Env) Exec(p *prog.Prog, cover, dedup bool) (output []byte, info []Cal
 		return
 	}
 
-	if env.flags&FlagSignal == 0 || p == nil {
+	if env.config.Flags&FlagSignal == 0 || p == nil {
 		return
 	}
 	info, err0 = env.readOutCoverage(p)
@@ -354,26 +371,25 @@ func closeMapping(f *os.File, mem []byte) error {
 
 type command struct {
 	pid      int
-	timeout  time.Duration
+	config   Config
 	cmd      *exec.Cmd
-	flags    uint64
 	dir      string
 	readDone chan []byte
+	exited   chan struct{}
 	inrp     *os.File
 	outwp    *os.File
 }
 
-func makeCommand(pid int, bin []string, timeout time.Duration, flags uint64, inFile *os.File, outFile *os.File) (*command, error) {
+func makeCommand(pid int, bin []string, config Config, inFile *os.File, outFile *os.File) (*command, error) {
 	dir, err := ioutil.TempDir("./", "syzkaller-testdir")
 	if err != nil {
 		return nil, fmt.Errorf("failed to create temp dir: %v", err)
 	}
 
 	c := &command{
-		pid:     pid,
-		timeout: timeout,
-		flags:   flags,
-		dir:     dir,
+		pid:    pid,
+		config: config,
+		dir:    dir,
 	}
 	defer func() {
 		if c != nil {
@@ -381,7 +397,7 @@ func makeCommand(pid int, bin []string, timeout time.Duration, flags uint64, inF
 		}
 	}()
 
-	if flags&(FlagSandboxSetuid|FlagSandboxNamespace) != 0 {
+	if config.Flags&(FlagSandboxSetuid|FlagSandboxNamespace) != 0 {
 		if err := os.Chmod(dir, 0777); err != nil {
 			return nil, fmt.Errorf("failed to chmod temp dir: %v", err)
 		}
@@ -411,12 +427,13 @@ func makeCommand(pid int, bin []string, timeout time.Duration, flags uint64, inF
 	c.outwp = outwp
 
 	c.readDone = make(chan []byte, 1)
+	c.exited = make(chan struct{})
 
 	cmd := exec.Command(bin[0], bin[1:]...)
 	cmd.ExtraFiles = []*os.File{inFile, outFile, outrp, inwp}
 	cmd.Env = []string{}
 	cmd.Dir = dir
-	if flags&FlagDebug == 0 {
+	if config.Flags&FlagDebug == 0 {
 		cmd.Stdout = wp
 		cmd.Stderr = wp
 		go func(c *command) {
@@ -463,8 +480,8 @@ func makeCommand(pid int, bin []string, timeout time.Duration, flags uint64, inF
 
 func (c *command) close() {
 	if c.cmd != nil {
-		c.kill()
-		c.cmd.Wait()
+		c.abort()
+		c.wait()
 	}
 	fileutil.UmountAll(c.dir)
 	os.RemoveAll(c.dir)
@@ -489,10 +506,10 @@ func (c *command) waitServing() error {
 	case err := <-read:
 		timeout.Stop()
 		if err != nil {
-			c.kill()
+			c.abort()
 			output := <-c.readDone
 			err = fmt.Errorf("executor is not serving: %v\n%s", err, output)
-			c.cmd.Wait()
+			c.wait()
 			if c.cmd.ProcessState != nil {
 				sys := c.cmd.ProcessState.Sys()
 				if ws, ok := sys.(syscall.WaitStatus); ok {
@@ -509,8 +526,32 @@ func (c *command) waitServing() error {
 	}
 }
 
-func (c *command) kill() {
-	syscall.Kill(c.cmd.Process.Pid, syscall.SIGKILL)
+// abort sends the abort signal to the command and then SIGKILL if wait doesn't
+// return within 5s.
+func (c *command) abort() {
+	syscall.Kill(c.cmd.Process.Pid, c.config.AbortSignal)
+	if c.config.AbortSignal != syscall.SIGKILL {
+		go func() {
+			t := time.NewTimer(5 * time.Second)
+			select {
+			case <-t.C:
+				syscall.Kill(c.cmd.Process.Pid, syscall.SIGKILL)
+			case <-c.exited:
+				t.Stop()
+			}
+		}()
+	}
+}
+
+func (c *command) wait() error {
+	err := c.cmd.Wait()
+	select {
+	case <-c.exited:
+		// c.exited closed by an earlier call to wait.
+	default:
+		close(c.exited)
+	}
+	return err
 }
 
 func (c *command) exec(cover, dedup bool) (output []byte, failed, hanged, restart bool, err0 error) {
@@ -529,10 +570,10 @@ func (c *command) exec(cover, dedup bool) (output []byte, failed, hanged, restar
 	done := make(chan bool)
 	hang := make(chan bool)
 	go func() {
-		t := time.NewTimer(c.timeout)
+		t := time.NewTimer(c.config.Timeout)
 		select {
 		case <-t.C:
-			c.kill()
+			c.abort()
 			hang <- true
 		case <-done:
 			t.Stop()
@@ -556,9 +597,9 @@ func (c *command) exec(cover, dedup bool) (output []byte, failed, hanged, restar
 		status = int(flags[0])
 	}
 	err0 = fmt.Errorf("executor did not answer")
-	c.kill()
+	c.abort()
 	output = <-c.readDone
-	if err := c.cmd.Wait(); <-hang && err != nil {
+	if err := c.wait(); <-hang && err != nil {
 		hanged = true
 		output = append(output, []byte(err.Error())...)
 		output = append(output, '\n')
diff --git a/syz-fuzzer/fuzzer.go b/syz-fuzzer/fuzzer.go
index 6f5750d75..b9b878f68 100644
--- a/syz-fuzzer/fuzzer.go
+++ b/syz-fuzzer/fuzzer.go
@@ -181,14 +181,14 @@ func main() {
 
 	kmemleakInit()
 
-	flags, timeout, err := ipc.DefaultFlags()
+	config, err := ipc.DefaultConfig()
 	if err != nil {
 		panic(err)
 	}
 	if _, ok := calls[sys.CallMap["syz_emit_ethernet"]]; ok {
-		flags |= ipc.FlagEnableTun
+		config.Flags |= ipc.FlagEnableTun
 	}
-	noCover = flags&ipc.FlagSignal == 0
+	noCover = config.Flags&ipc.FlagSignal == 0
 	leakCallback := func() {
 		if atomic.LoadUint32(&allTriaged) != 0 {
 			// Scan for leaks once in a while (it is damn slow).
@@ -203,7 +203,7 @@ func main() {
 	needPoll <- struct{}{}
 	envs := make([]*ipc.Env, *flagProcs)
 	for pid := 0; pid < *flagProcs; pid++ {
-		env, err := ipc.MakeEnv(*flagExecutor, timeout, flags, pid)
+		env, err := ipc.MakeEnv(*flagExecutor, pid, config)
 		if err != nil {
 			panic(err)
 		}
diff --git a/tools/syz-execprog/execprog.go b/tools/syz-execprog/execprog.go
index 7e0a25d75..1992279a7 100644
--- a/tools/syz-execprog/execprog.go
+++ b/tools/syz-execprog/execprog.go
@@ -56,14 +56,14 @@ func main() {
 		return
 	}
 
-	flags, timeout, err := ipc.DefaultFlags()
+	config, err := ipc.DefaultConfig()
 	if err != nil {
 		Fatalf("%v", err)
 	}
-	needCover := flags&ipc.FlagSignal != 0
+	needCover := config.Flags&ipc.FlagSignal != 0
 	dedupCover := true
 	if *flagCoverFile != "" {
-		flags |= ipc.FlagSignal
+		config.Flags |= ipc.FlagSignal
 		needCover = true
 		dedupCover = false
 	}
@@ -75,7 +75,7 @@ func main() {
 		}
 	}
 	if handled["syz_emit_ethernet"] {
-		flags |= ipc.FlagEnableTun
+		config.Flags |= ipc.FlagEnableTun
 	}
 
 	var wg sync.WaitGroup
@@ -89,7 +89,7 @@ func main() {
 		pid := p
 		go func() {
 			defer wg.Done()
-			env, err := ipc.MakeEnv(*flagExecutor, timeout, flags, pid)
+			env, err := ipc.MakeEnv(*flagExecutor, pid, config)
 			if err != nil {
 				Fatalf("failed to create ipc env: %v", err)
 			}
@@ -126,7 +126,7 @@ func main() {
 					if failed {
 						fmt.Printf("BUG: executor-detected bug:\n%s", output)
 					}
-					if flags&ipc.FlagDebug != 0 || err != nil {
+					if config.Flags&ipc.FlagDebug != 0 || err != nil {
 						fmt.Printf("result: failed=%v hanged=%v err=%v\n\n%s", failed, hanged, err, output)
 					}
 					if *flagCoverFile != "" {
diff --git a/tools/syz-stress/stress.go b/tools/syz-stress/stress.go
index 043af44fd..87f016504 100644
--- a/tools/syz-stress/stress.go
+++ b/tools/syz-stress/stress.go
@@ -50,7 +50,7 @@ func main() {
 	prios := prog.CalculatePriorities(corpus)
 	ct := prog.BuildChoiceTable(prios, calls)
 
-	flags, timeout, err := ipc.DefaultFlags()
+	config, err := ipc.DefaultConfig()
 	if err != nil {
 		Fatalf("%v", err)
 	}
@@ -58,7 +58,7 @@ func main() {
 	for pid := 0; pid < *flagProcs; pid++ {
 		pid := pid
 		go func() {
-			env, err := ipc.MakeEnv(*flagExecutor, timeout, flags, pid)
+			env, err := ipc.MakeEnv(*flagExecutor, pid, config)
 			if err != nil {
 				Fatalf("failed to create execution environment: %v", err)
 			}