From 0165a4b2e4384f44c8042dcacaf52563ec74bb8c Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Tue, 10 Nov 2015 20:30:50 +0100 Subject: use fork server in executor This avoids exec per test. Also allows to pre-map shared memory regions. And will allow to pre-map coverage regions, etc. Seems to work already, but probably there are still some bugs. --- executor/executor.cc | 113 +++++++++++++++------ fuzzer/fuzzer.go | 2 +- ipc/ipc.go | 239 ++++++++++++++++++++++++++++++++++++++++++++- tools/execprog/execprog.go | 61 ++++++------ 4 files changed, 353 insertions(+), 62 deletions(-) diff --git a/executor/executor.cc b/executor/executor.cc index e126d466d..2c2aa9841 100644 --- a/executor/executor.cc +++ b/executor/executor.cc @@ -11,11 +11,14 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -24,6 +27,8 @@ const int kInFd = 3; const int kOutFd = 4; +const int kInPipeFd = 5; +const int kOutPipeFd = 6; const int kCoverFd = 5; const int kMaxInput = 1 << 20; const int kMaxOutput = 16 << 20; @@ -39,6 +44,9 @@ const uint64_t arg_const = 0; const uint64_t arg_result = 1; const uint64_t arg_data = 2; +const int kFailStatus = 67; +const int kErrorStatus = 68; + // We use the default value instead of results of failed syscalls. // -1 is an invalid fd and an invalid address and deterministic, // so good enough for our purposes. @@ -89,6 +97,7 @@ __attribute__((noreturn)) void fail(const char* msg, ...); __attribute__((noreturn)) void error(const char* msg, ...); __attribute__((noreturn)) void exitf(const char* msg, ...); void debug(const char* msg, ...); +void execute_one(); uint64_t read_input(uint64_t** input_posp, bool peek = false); uint64_t read_arg(uint64_t** input_posp); uint64_t read_result(uint64_t** input_posp); @@ -112,35 +121,84 @@ int main() fail("mmap of input file failed"); if (mmap(&output_data[0], kMaxOutput, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, kOutFd, 0) != &output_data[0]) fail("mmap of output file failed"); -retry: - uint64_t* input_pos = (uint64_t*)&input_data[0]; - uint64_t flags = read_input(&input_pos); + char cwdbuf[64 << 10]; + char* cwd = getcwd(cwdbuf, sizeof(cwdbuf)); + + sigset_t sigchldset; + sigemptyset(&sigchldset); + sigaddset(&sigchldset, SIGCHLD); + if (sigprocmask(SIG_BLOCK, &sigchldset, NULL)) + fail("sigprocmask failed"); + + uint64_t flags = *(uint64_t*)input_data; flag_debug = flags & (1 << 0); flag_cover = flags & (1 << 1); flag_threaded = flags & (1 << 2); flag_collide = flags & (1 << 3); flag_deduplicate = flags & (1 << 4); flag_drop_privs = flags & (1 << 5); - output_pos = (uint32_t*)&output_data[0]; - write_output(0); // Number of executed syscalls (updated later). - if (flag_collide) flag_threaded = true; - if (!collide) { - cover_open(); - if (!flag_threaded) - cover_init(&threads[0]); - - if (flag_drop_privs) { - // TODO: 65534 is meant to be nobody - if (setgroups(0, NULL)) - fail("failed to setgroups"); - if (setresgid(65534, 65534, 65534)) - fail("failed to setresgid"); - if (setresuid(65534, 65534, 65534)) - fail("failed to setresuid"); + + cover_open(); + + for (;;) { + char tmp; + if (read(kInPipeFd, &tmp, 1) != 1) + fail("control pipe read failed"); + // The dir may have been recreated. + if (chdir(cwd)) + fail("failed to chdir"); + + int pid = fork(); + if (pid < 0) + fail("fork failed"); + if (pid == 0) { + setpgid(0, 0); + if (flag_drop_privs) { + // TODO: 65534 is meant to be nobody + if (setgroups(0, NULL)) + fail("failed to setgroups"); + if (setresgid(65534, 65534, 65534)) + fail("failed to setresgid"); + if (setresuid(65534, 65534, 65534)) + fail("failed to setresuid"); + } + execute_one(); + debug("exiting\n"); + return 0; } + + timespec ts = {}; + ts.tv_sec = 5; + ts.tv_nsec = 0; + if (sigtimedwait(&sigchldset, NULL, &ts) < 0) { + kill(-pid, SIGKILL); + kill(pid, SIGKILL); + } + int status = 0; + if (waitpid(pid, &status, 0) != pid) + fail("waitpid failed"); + status = WEXITSTATUS(status); + if (status == kFailStatus) + fail("child failed"); + if (status == kErrorStatus) + error("child errored"); + if (write(kOutPipeFd, &tmp, 1) != 1) + fail("control pipe write failed"); } +} + +void execute_one() +{ +retry: + uint64_t* input_pos = (uint64_t*)&input_data[0]; + read_input(&input_pos); // flags + output_pos = (uint32_t*)&output_data[0]; + write_output(0); // Number of executed syscalls (updated later). + + if (!collide && !flag_threaded) + cover_init(&threads[0]); int call_index = 0; for (int n = 0;; n++) { @@ -236,16 +294,11 @@ retry: } } - if (flag_collide) { - if (!collide) { - debug("enabling collider\n"); - collide = true; - goto retry; - } + if (flag_collide && !collide) { + debug("enabling collider\n"); + collide = true; + goto retry; } - - debug("exiting\n"); - return 0; } thread_t* schedule_call(int n, int call_index, int call_num, uint64_t num_args, uint64_t* args, uint64_t* pos) @@ -537,7 +590,7 @@ void fail(const char* msg, ...) vfprintf(stderr, msg, args); va_end(args); fprintf(stderr, " (errno %d)\n", e); - exit(67); + exit(kFailStatus); } // kernel error (e.g. wrong syscall return value) @@ -549,7 +602,7 @@ void error(const char* msg, ...) vfprintf(stderr, msg, args); va_end(args); fprintf(stderr, "\n"); - exit(68); + exit(kErrorStatus); } // just exit (e.g. due to temporal ENOMEM error) diff --git a/fuzzer/fuzzer.go b/fuzzer/fuzzer.go index cb78d848d..cdcbcfca6 100644 --- a/fuzzer/fuzzer.go +++ b/fuzzer/fuzzer.go @@ -109,7 +109,7 @@ func main() { if !*flagNoCover { flags |= ipc.FlagCover | ipc.FlagDedupCover } - env, err := ipc.MakeEnv(*flagExecutor, 4*time.Second, flags) + env, err := ipc.MakeEnv(*flagExecutor, 10*time.Second, flags) if err != nil { panic(err) } diff --git a/ipc/ipc.go b/ipc/ipc.go index 4c881228b..7673179a7 100644 --- a/ipc/ipc.go +++ b/ipc/ipc.go @@ -22,6 +22,7 @@ type Env struct { In []byte Out []byte + cmd *command inFile *os.File outFile *os.File bin []string @@ -74,12 +75,19 @@ func MakeEnv(bin string, timeout time.Duration, flags uint64) (*Env, error) { if len(env.bin) == 0 { return nil, fmt.Errorf("binary is empty string") } + env.bin[0], err = filepath.Abs(env.bin[0]) // we are going to chdir + if err != nil { + return nil, fmt.Errorf("filepath.Abs failed: %v", err) + } inf = nil outf = nil return env, nil } func (env *Env) Close() error { + if env.cmd != nil { + env.cmd.close() + } err1 := closeMapping(env.inFile, env.In) err2 := closeMapping(env.outFile, env.Out) switch { @@ -116,8 +124,16 @@ func (env *Env) Exec(p *prog.Prog) (output, strace []byte, cov [][]uint32, faile } } - output, strace, failed, hanged, err0 = env.execBin() + if env.cmd == nil { + env.cmd, err0 = makeCommand(env.bin, env.timeout, env.flags, env.inFile, env.outFile) + if err0 != nil { + return + } + } + output, strace, failed, hanged, err0 = env.cmd.exec() if err0 != nil { + env.cmd.close() + env.cmd = nil return } @@ -172,12 +188,15 @@ func (env *Env) Exec(p *prog.Prog) (output, strace []byte, cov [][]uint32, faile return } +/* func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 error) { dir, err := ioutil.TempDir("./", "syzkaller-testdir") if err != nil { err0 = fmt.Errorf("failed to create temp dir: %v", err) return } + + // Output capture pipe. defer os.RemoveAll(dir) rp, wp, err := os.Pipe() if err != nil { @@ -186,6 +205,25 @@ func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 erro } defer rp.Close() defer wp.Close() + + // Input command pipe. + inrp, inwp, err := os.Pipe() + if err != nil { + err0 = fmt.Errorf("failed to create pipe: %v", err) + return + } + defer inrp.Close() + defer inwp.Close() + + // Output command pipe. + outrp, outwp, err := os.Pipe() + if err != nil { + err0 = fmt.Errorf("failed to create pipe: %v", err) + return + } + defer outrp.Close() + defer outwp.Close() + cmd := exec.Command(env.bin[0], env.bin[1:]...) traceFile := "" if env.flags&FlagStrace != 0 { @@ -204,7 +242,7 @@ func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 erro } cmd = exec.Command("strace", args...) } - cmd.ExtraFiles = append(cmd.ExtraFiles, env.inFile, env.outFile) + cmd.ExtraFiles = append(cmd.ExtraFiles, env.inFile, env.outFile, outrp, inwp) cmd.Env = []string{} cmd.Dir = dir if env.flags&FlagDebug == 0 { @@ -220,11 +258,17 @@ func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 erro } else { cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} } + if _, err := outwp.Write([]byte{0}); err != nil { + err0 = fmt.Errorf("failed to write control pipe: %v", err) + return + } if err := cmd.Start(); err != nil { err0 = fmt.Errorf("failed to start executor binary: %v", err) return } wp.Close() + outrp.Close() + inwp.Close() done := make(chan bool) hang := make(chan bool) go func() { @@ -234,6 +278,7 @@ func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 erro // We started the process in its own process group and now kill the whole group. // This solves a potential problem with strace: // if we kill just strace, executor still runs and ReadAll below hangs. + fmt.Printf("KILLING %v\n", cmd.Process.Pid) syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) syscall.Kill(cmd.Process.Pid, syscall.SIGKILL) @@ -244,6 +289,11 @@ func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 erro hang <- false } }() + var tmp [1]byte + if n, err := inrp.Read(tmp[:]); n != 1 || err != nil { + err0 = fmt.Errorf("failed to read control pipe: %v", err) + return + } output, err = ioutil.ReadAll(rp) readErr := err close(done) @@ -281,6 +331,7 @@ func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 erro } return } +*/ func createMapping(size int) (f *os.File, mem []byte, err error) { f, err = ioutil.TempFile("./", "syzkaller-shm") @@ -326,3 +377,187 @@ func closeMapping(f *os.File, mem []byte) error { return nil } } + +type command struct { + timeout time.Duration + cmd *exec.Cmd + dir string + rp *os.File + inrp *os.File + outwp *os.File +} + +func makeCommand(bin []string, timeout time.Duration, flags uint64, inFile *os.File, outFile *os.File) (*command, error) { + dir, err := ioutil.TempDir("./", "syzkaller-testdir") + if err != nil { + return nil, fmt.Errorf("failed to create temp dir: %v", err) + } + if err := os.Chmod(dir, 0777); err != nil { + return nil, fmt.Errorf("failed to chmod temp dir: %v", err) + } + + c := &command{timeout: timeout, dir: dir} + defer func() { + if c != nil { + c.close() + } + }() + + // Output capture pipe. + rp, wp, err := os.Pipe() + if err != nil { + return nil, fmt.Errorf("failed to create pipe: %v", err) + } + defer wp.Close() + c.rp = rp + + // Input command pipe. + inrp, inwp, err := os.Pipe() + if err != nil { + return nil, fmt.Errorf("failed to create pipe: %v", err) + } + defer inwp.Close() + c.inrp = inrp + + // Output command pipe. + outrp, outwp, err := os.Pipe() + if err != nil { + return nil, fmt.Errorf("failed to create pipe: %v", err) + } + defer outrp.Close() + c.outwp = outwp + + cmd := exec.Command(bin[0], bin[1:]...) + /* + traceFile := "" + if flags&FlagStrace != 0 { + f, err := ioutil.TempFile("./", "syzkaller-strace") + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %v", err) + } + f.Close() + defer os.Remove(f.Name()) + traceFile, _ = filepath.Abs(f.Name()) + args := []string{"-s", "8", "-o", traceFile} + args = append(args, env.bin...) + if env.flags&FlagThreaded != 0 { + args = append([]string{"-f"}, args...) + } + cmd = exec.Command("strace", args...) + } + */ + cmd.ExtraFiles = []*os.File{inFile, outFile, outrp, inwp} + cmd.Env = []string{} + cmd.Dir = dir + if flags&FlagDebug == 0 { + cmd.Stdout = wp + cmd.Stderr = wp + } else { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stdout + } + if syscall.Getuid() == 0 { + // Running under root, more isolation is possible. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Cloneflags: syscall.CLONE_NEWNS} + } else { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + } + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start executor binary: %v", err) + } + c.cmd = cmd + tmp := c + c = nil // disable defer above + return tmp, nil +} + +func (c *command) close() { + c.kill() + c.cmd.Wait() + os.RemoveAll(c.dir) + if c.rp != nil { + c.rp.Close() + } + if c.inrp != nil { + c.inrp.Close() + } + if c.outwp != nil { + c.outwp.Close() + } +} + +func (c *command) kill() { + // We started the process in its own process group and now kill the whole group. + // This solves a potential problem with strace: + // if we kill just strace, executor still runs and ReadAll below hangs. + syscall.Kill(-c.cmd.Process.Pid, syscall.SIGKILL) + syscall.Kill(c.cmd.Process.Pid, syscall.SIGKILL) +} + +func (c *command) exec() (output, strace []byte, failed, hanged bool, err0 error) { + var tmp [1]byte + if _, err := c.outwp.Write(tmp[:]); err != nil { + err0 = fmt.Errorf("failed to write control pipe: %v", err) + return + } + done := make(chan bool) + hang := make(chan bool) + go func() { + t := time.NewTimer(c.timeout) + select { + case <-t.C: + c.kill() + hang <- true + case <-done: + t.Stop() + hang <- false + } + }() + //!!! handle c.rp overflow + _, readErr := c.inrp.Read(tmp[:]) + close(done) + os.RemoveAll(c.dir) + if err := os.Mkdir(c.dir, 0777); err != nil { + <-hang + err0 = fmt.Errorf("failed to create temp dir: %v", err) + return + } + if readErr == nil { + <-hang + return + } + err0 = fmt.Errorf("executor did not answer") + c.kill() + var err error + output, err = ioutil.ReadAll(c.rp) + if err = c.cmd.Wait(); <-hang && err != nil { + hanged = true + } + if err != nil { + output = append(output, []byte(err.Error())...) + output = append(output, '\n') + } + if c.cmd.ProcessState != nil { + sys := c.cmd.ProcessState.Sys() + if ws, ok := sys.(syscall.WaitStatus); ok { + // Magic values returned by executor. + if ws.ExitStatus() == 67 { + err0 = fmt.Errorf("executor failed: %s", output) + return + } + if ws.ExitStatus() == 68 { + failed = true + } + } + } + /* + if traceFile != "" { + strace, err = ioutil.ReadFile(traceFile) + if err != nil { + err0 = fmt.Errorf("failed to read strace output: %v", err) + return + } + } + */ + return +} diff --git a/tools/execprog/execprog.go b/tools/execprog/execprog.go index 471808b6c..f85925125 100644 --- a/tools/execprog/execprog.go +++ b/tools/execprog/execprog.go @@ -24,17 +24,18 @@ import ( ) var ( - flagExecutor = flag.String("executor", "", "path to executor binary") - flagThreaded = flag.Bool("threaded", true, "use threaded mode in executor") - flagCollide = flag.Bool("collide", true, "collide syscalls to provoke data races") - flagDebug = flag.Bool("debug", false, "debug output from executor") - flagStrace = flag.Bool("strace", false, "run executor under strace") - flagCover = flag.String("cover", "", "collect coverage and write to the file") - flagNobody = flag.Bool("nobody", true, "impersonate into nobody") - flagDedup = flag.Bool("dedup", false, "deduplicate coverage in executor") - flagLoop = flag.Bool("loop", false, "execute programs in a loop") - flagProcs = flag.Int("procs", 1, "number of parallel processes to execute programs") - flagTimeout = flag.Duration("timeout", 5*time.Second, "execution timeout") + flagExecutor = flag.String("executor", "", "path to executor binary") + flagThreaded = flag.Bool("threaded", true, "use threaded mode in executor") + flagCollide = flag.Bool("collide", true, "collide syscalls to provoke data races") + flagDebug = flag.Bool("debug", false, "debug output from executor") + flagStrace = flag.Bool("strace", false, "run executor under strace") + flagCover = flag.Bool("cover", true, "collect coverage") + flagCoverFile = flag.String("coverfile", "", "write coverage to the file") + flagNobody = flag.Bool("nobody", true, "impersonate into nobody") + flagDedup = flag.Bool("dedup", false, "deduplicate coverage in executor") + flagLoop = flag.Bool("loop", false, "execute programs in a loop") + flagProcs = flag.Int("procs", 1, "number of parallel processes to execute programs") + flagTimeout = flag.Duration("timeout", 10*time.Second, "execution timeout") ) func main() { @@ -67,7 +68,7 @@ func main() { if *flagStrace { flags |= ipc.FlagStrace } - if *flagCover != "" { + if *flagCover || *flagCoverFile != "" { flags |= ipc.FlagCover } if *flagDedup { @@ -98,28 +99,30 @@ func main() { } p := progs[idx%len(progs)] output, strace, cov, failed, hanged, err := env.Exec(p) - if *flagDebug { + if *flagDebug || err != nil { fmt.Printf("result: failed=%v hanged=%v err=%v\n\n%s", failed, hanged, err, output) } if *flagStrace { fmt.Printf("strace output:\n%s", strace) } - // Coverage is dumped in sanitizer format. - // github.com/google/sanitizers/tools/sancov command can be used to dump PCs, - // then they can be piped via addr2line to symbolize. - for i, c := range cov { - fmt.Printf("call #%v: coverage %v\n", i, len(c)) - if len(c) == 0 { - continue - } - buf := new(bytes.Buffer) - binary.Write(buf, binary.LittleEndian, uint64(0xC0BFFFFFFFFFFF64)) - for _, pc := range c { - binary.Write(buf, binary.LittleEndian, cover.RestorePC(pc)) - } - err := ioutil.WriteFile(fmt.Sprintf("%v.%v", *flagCover, i), buf.Bytes(), 0660) - if err != nil { - log.Fatalf("failed to write coverage file: %v", err) + if *flagCoverFile != "" { + // Coverage is dumped in sanitizer format. + // github.com/google/sanitizers/tools/sancov command can be used to dump PCs, + // then they can be piped via addr2line to symbolize. + for i, c := range cov { + fmt.Printf("call #%v: coverage %v\n", i, len(c)) + if len(c) == 0 { + continue + } + buf := new(bytes.Buffer) + binary.Write(buf, binary.LittleEndian, uint64(0xC0BFFFFFFFFFFF64)) + for _, pc := range c { + binary.Write(buf, binary.LittleEndian, cover.RestorePC(pc)) + } + err := ioutil.WriteFile(fmt.Sprintf("%v.%v", *flagCoverFile, i), buf.Bytes(), 0660) + if err != nil { + log.Fatalf("failed to write coverage file: %v", err) + } } } } -- cgit mrf-deployment