From 8f58526cb8e159721342f3880658a1a2547adab8 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Fri, 7 Apr 2017 10:24:54 +0200 Subject: all: add fault injection capability Systematically inject faults during smashing. Requires kernel patch: "fault-inject: support systematic fault injection" (currently in linux-next). --- executor/executor.cc | 55 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 7 deletions(-) (limited to 'executor') diff --git a/executor/executor.cc b/executor/executor.cc index 02792f82f..bdda612ca 100644 --- a/executor/executor.cc +++ b/executor/executor.cc @@ -76,9 +76,14 @@ bool flag_collide; bool flag_sandbox_privs; sandbox_type flag_sandbox; bool flag_enable_tun; +bool flag_enable_fault_injection; bool flag_collect_cover; bool flag_dedup_cover; +// Inject fault into flag_fault_nth-th operation in flag_fault_call-th syscall. +bool flag_inject_fault; +int flag_fault_call; +int flag_fault_nth; __attribute__((aligned(64 << 10))) char input_data[kMaxInput]; uint32_t* output_data; @@ -111,6 +116,7 @@ struct thread_t { uint64_t res; uint64_t reserrno; uint64_t cover_size; + bool fault_injected; int cover_fd; }; @@ -135,7 +141,6 @@ void execute_call(thread_t* th); void handle_completion(thread_t* th); void thread_create(thread_t* th, int id); void* worker_thread(void* arg); -bool write_file(const char* file, const char* what, ...); void cover_open(); void cover_enable(thread_t* th); void cover_reset(thread_t* th); @@ -176,8 +181,9 @@ int main(int argc, char** argv) if (!flag_threaded) flag_collide = false; flag_enable_tun = flags & (1 << 6); - uint64_t executor_pid = *((uint64_t*)input_data + 1); + flag_enable_fault_injection = flags & (1 << 7); + uint64_t executor_pid = *((uint64_t*)input_data + 1); cover_open(); setup_main_process(); @@ -240,11 +246,16 @@ void loop() // TODO: consider moving the read into the child. // Potentially it can speed up things a bit -- when the read finishes // we already have a forked worker process. - char flags = 0; - if (read(kInPipeFd, &flags, 1) != 1) + uint64_t in_cmd[3] = {}; + if (read(kInPipeFd, &in_cmd[0], sizeof(in_cmd)) != (ssize_t)sizeof(in_cmd)) fail("control pipe read failed"); - flag_collect_cover = flags & (1 << 0); - flag_dedup_cover = flags & (1 << 1); + flag_collect_cover = in_cmd[0] & (1 << 0); + flag_dedup_cover = in_cmd[0] & (1 << 1); + flag_inject_fault = in_cmd[0] & (1 << 2); + flag_fault_call = in_cmd[1]; + flag_fault_nth = in_cmd[2]; + debug("exec opts: cover=%d dedup=%d fault=%d/%d/%d\n", flag_collect_cover, flag_dedup_cover, + flag_inject_fault, flag_fault_call, flag_fault_nth); int pid = fork(); if (pid < 0) @@ -482,7 +493,7 @@ retry: } } - if (flag_collide && !collide) { + if (flag_collide && !flag_inject_fault && !collide) { debug("enabling collider\n"); collide = true; goto retry; @@ -561,6 +572,7 @@ void handle_completion(thread_t* th) write_output(th->call_num); uint32_t reserrno = th->res != (uint64_t)-1 ? 0 : th->reserrno; write_output(reserrno); + write_output(th->fault_injected); uint32_t* signal_count_pos = write_output(0); // filled in later uint32_t* cover_count_pos = write_output(0); // filled in later @@ -651,10 +663,39 @@ void execute_call(thread_t* th) } debug(")\n"); + int fail_fd = -1; + if (flag_inject_fault && th->call_index == flag_fault_call) { + if (collide) + fail("both collide and fault injection are enabled"); + debug("injecting fault into %d-th operation\n", flag_fault_nth); + char buf[128]; + sprintf(buf, "/proc/self/task/%d/fail-nth", (int)syscall(SYS_gettid)); + fail_fd = open(buf, O_RDWR); + if (fail_fd == -1) + fail("failed to open /proc/self/task/tid/fail-nth"); + sprintf(buf, "%d", flag_fault_nth + 1); + if (write(fail_fd, buf, strlen(buf)) != (ssize_t)strlen(buf)) + fail("failed to write /proc/self/task/tid/fail-nth"); + } + cover_reset(th); th->res = execute_syscall(call->sys_nr, th->args[0], th->args[1], th->args[2], th->args[3], th->args[4], th->args[5], th->args[6], th->args[7], th->args[8]); th->reserrno = errno; th->cover_size = cover_read(th); + th->fault_injected = false; + + if (flag_inject_fault && th->call_index == flag_fault_call) { + char buf[16]; + int n = read(fail_fd, buf, sizeof(buf) - 1); + if (n <= 0) + fail("failed to read /proc/self/task/tid/fail-nth"); + th->fault_injected = n == 2 && buf[0] == '0' && buf[1] == '\n'; + buf[0] = '0'; + if (write(fail_fd, buf, 1) != 1) + fail("failed to write /proc/self/task/tid/fail-nth"); + close(fail_fd); + debug("fault injected: %d\n", th->fault_injected); + } if (th->res == (uint64_t)-1) debug("#%d: %s = errno(%ld)\n", th->id, call->name, th->reserrno); -- cgit mrf-deployment From 220dc49106d66ff912db835004c88f8c9e2d1707 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Thu, 25 May 2017 16:07:10 +0200 Subject: csource: reproduce crashes with fault injection --- csource/common.go | 33 ++++++++++++++++++++++++++------- csource/csource.go | 27 +++++++++++++++++++-------- csource/csource_test.go | 20 +++++++++++--------- executor/common.h | 33 ++++++++++++++++++++++++++------- executor/executor.cc | 9 +-------- prog/parse.go | 40 ++++++++++++++++++++++++++++------------ prog/parse_test.go | 34 ++++++++++++++++++++++++++++++++++ repro/repro.go | 23 ++++++++++++++++++----- tools/syz-prog2c/prog2c.go | 29 +++++++++++++++++------------ 9 files changed, 180 insertions(+), 68 deletions(-) (limited to 'executor') diff --git a/csource/common.go b/csource/common.go index 240dff6c8..25d062821 100644 --- a/csource/common.go +++ b/csource/common.go @@ -1602,13 +1602,7 @@ static int do_sandbox_setuid(int executor_pid, bool enable_tun) } #endif -#if defined(SYZ_EXECUTOR) || defined(SYZ_SANDBOX_NAMESPACE) -static int real_uid; -static int real_gid; -static int epid; -static bool etun; -__attribute__((aligned(64 << 10))) static char sandbox_stack[1 << 20]; - +#if defined(SYZ_EXECUTOR) || defined(SYZ_SANDBOX_NAMESPACE) || defined(SYZ_FAULT_INJECTION) static bool write_file(const char* file, const char* what, ...) { char buf[1024]; @@ -1629,6 +1623,14 @@ static bool write_file(const char* file, const char* what, ...) close(fd); return true; } +#endif + +#if defined(SYZ_EXECUTOR) || defined(SYZ_SANDBOX_NAMESPACE) +static int real_uid; +static int real_gid; +static int epid; +static bool etun; +__attribute__((aligned(64 << 10))) static char sandbox_stack[1 << 20]; static int namespace_sandbox_proc(void* arg) { @@ -1785,6 +1787,23 @@ static uint64_t current_time_ms() } #endif +#if defined(SYZ_EXECUTOR) || defined(SYZ_FAULT_INJECTION) +static int inject_fault(int nth) +{ + int fd; + char buf[128]; + + sprintf(buf, "/proc/self/task/%d/fail-nth", (int)syscall(SYS_gettid)); + fd = open(buf, O_RDWR); + if (fd == -1) + fail("failed to open /proc/self/task/tid/fail-nth"); + sprintf(buf, "%d", nth + 1); + if (write(fd, buf, strlen(buf)) != (ssize_t)strlen(buf)) + fail("failed to write /proc/self/task/tid/fail-nth"); + return fd; +} +#endif + #if defined(SYZ_REPEAT) static void test(); diff --git a/csource/csource.go b/csource/csource.go index 959538dbb..ab585ebec 100644 --- a/csource/csource.go +++ b/csource/csource.go @@ -22,12 +22,15 @@ import ( ) type Options struct { - Threaded bool - Collide bool - Repeat bool - Procs int - Sandbox string - Repro bool // generate code for use with repro package + Threaded bool + Collide bool + Repeat bool + Procs int + Sandbox string + Fault bool // inject fault into FaultCall/FaultNth + FaultCall int + FaultNth int + Repro bool // generate code for use with repro package } func Write(p *prog.Prog, opts Options) ([]byte, error) { @@ -65,7 +68,7 @@ func Write(p *prog.Prog, opts Options) ([]byte, error) { fmt.Fprint(w, hdr) fmt.Fprint(w, "\n") - calls, nvar := generateCalls(exec) + calls, nvar := generateCalls(exec, opts) fmt.Fprintf(w, "long r[%v];\n", nvar) if !opts.Repeat { @@ -161,7 +164,7 @@ func generateTestFunc(w io.Writer, opts Options, calls []string, name string) { } } -func generateCalls(exec []byte) ([]string, int) { +func generateCalls(exec []byte, opts Options) ([]string, int) { read := func() uintptr { if len(exec) < 8 { panic("exec program overflow") @@ -265,6 +268,11 @@ loop: default: // Normal syscall. newCall() + if opts.Fault && opts.FaultCall == len(calls) { + fmt.Fprintf(w, "\twrite_file(\"/sys/kernel/debug/failslab/ignore-gfp-wait\", \"N\");\n") + fmt.Fprintf(w, "\twrite_file(\"/sys/kernel/debug/fail_futex/ignore-private\", \"N\");\n") + fmt.Fprintf(w, "\tinject_fault(%v);\n", opts.FaultNth) + } meta := sys.Calls[instr] fmt.Fprintf(w, "\tr[%v] = execute_syscall(__NR_%v", n, meta.CallName) nargs := read() @@ -311,6 +319,9 @@ func preprocessCommonHeader(opts Options, handled map[string]int) (string, error if opts.Repeat { defines = append(defines, "SYZ_REPEAT") } + if opts.Fault { + defines = append(defines, "SYZ_FAULT_INJECTION") + } for name, _ := range handled { defines = append(defines, "__NR_"+name) } diff --git a/csource/csource_test.go b/csource/csource_test.go index 259b5689b..b5832ad8c 100644 --- a/csource/csource_test.go +++ b/csource/csource_test.go @@ -34,16 +34,18 @@ func allOptionsPermutations() []Options { for _, opt.Repro = range []bool{false, true} { for _, opt.Procs = range []int{1, 4} { for _, opt.Sandbox = range []string{"none", "setuid", "namespace"} { - if opt.Collide && !opt.Threaded { - continue + for _, opt.Fault = range []bool{false, true} { + if opt.Collide && !opt.Threaded { + continue + } + if !opt.Repeat && opt.Procs != 1 { + continue + } + if testing.Short() && opt.Procs != 1 { + continue + } + options = append(options, opt) } - if !opt.Repeat && opt.Procs != 1 { - continue - } - if testing.Short() && opt.Procs != 1 { - continue - } - options = append(options, opt) } } } diff --git a/executor/common.h b/executor/common.h index c32341f57..73a528471 100644 --- a/executor/common.h +++ b/executor/common.h @@ -699,13 +699,7 @@ static int do_sandbox_setuid(int executor_pid, bool enable_tun) } #endif -#if defined(SYZ_EXECUTOR) || defined(SYZ_SANDBOX_NAMESPACE) -static int real_uid; -static int real_gid; -static int epid; -static bool etun; -__attribute__((aligned(64 << 10))) static char sandbox_stack[1 << 20]; - +#if defined(SYZ_EXECUTOR) || defined(SYZ_SANDBOX_NAMESPACE) || defined(SYZ_FAULT_INJECTION) static bool write_file(const char* file, const char* what, ...) { char buf[1024]; @@ -726,6 +720,14 @@ static bool write_file(const char* file, const char* what, ...) close(fd); return true; } +#endif + +#if defined(SYZ_EXECUTOR) || defined(SYZ_SANDBOX_NAMESPACE) +static int real_uid; +static int real_gid; +static int epid; +static bool etun; +__attribute__((aligned(64 << 10))) static char sandbox_stack[1 << 20]; static int namespace_sandbox_proc(void* arg) { @@ -897,6 +899,23 @@ static uint64_t current_time_ms() } #endif +#if defined(SYZ_EXECUTOR) || defined(SYZ_FAULT_INJECTION) +static int inject_fault(int nth) +{ + int fd; + char buf[128]; + + sprintf(buf, "/proc/self/task/%d/fail-nth", (int)syscall(SYS_gettid)); + fd = open(buf, O_RDWR); + if (fd == -1) + fail("failed to open /proc/self/task/tid/fail-nth"); + sprintf(buf, "%d", nth + 1); + if (write(fd, buf, strlen(buf)) != (ssize_t)strlen(buf)) + fail("failed to write /proc/self/task/tid/fail-nth"); + return fd; +} +#endif + #if defined(SYZ_REPEAT) static void test(); diff --git a/executor/executor.cc b/executor/executor.cc index bdda612ca..cb31e0697 100644 --- a/executor/executor.cc +++ b/executor/executor.cc @@ -668,14 +668,7 @@ void execute_call(thread_t* th) if (collide) fail("both collide and fault injection are enabled"); debug("injecting fault into %d-th operation\n", flag_fault_nth); - char buf[128]; - sprintf(buf, "/proc/self/task/%d/fail-nth", (int)syscall(SYS_gettid)); - fail_fd = open(buf, O_RDWR); - if (fail_fd == -1) - fail("failed to open /proc/self/task/tid/fail-nth"); - sprintf(buf, "%d", flag_fault_nth + 1); - if (write(fail_fd, buf, strlen(buf)) != (ssize_t)strlen(buf)) - fail("failed to write /proc/self/task/tid/fail-nth"); + fail_fd = inject_fault(flag_fault_nth); } cover_reset(th); diff --git a/prog/parse.go b/prog/parse.go index e4a4e15ad..2044b1c9f 100644 --- a/prog/parse.go +++ b/prog/parse.go @@ -10,10 +10,13 @@ import ( // LogEntry describes one program in execution log. type LogEntry struct { - P *Prog - Proc int // index of parallel proc - Start int // start offset in log - End int // end offset in log + P *Prog + Proc int // index of parallel proc + Start int // start offset in log + End int // end offset in log + Fault bool // program was executed with fault injection in FaultCall/FaultNth + FaultCall int + FaultNth int } func ParseLog(data []byte) []*LogEntry { @@ -30,22 +33,21 @@ func ParseLog(data []byte) []*LogEntry { line := data[pos : nl+1] pos0 := pos pos = nl + 1 - const delim = "executing program " - if delimPos := bytes.Index(line, []byte(delim)); delimPos != -1 { + + if proc, ok := extractInt(line, "executing program "); ok { if ent.P != nil && len(ent.P.Calls) != 0 { ent.End = pos0 entries = append(entries, ent) } - procStart := delimPos + len(delim) - procEnd := procStart - for procEnd != len(line) && line[procEnd] >= '0' && line[procEnd] <= '9' { - procEnd++ - } - proc, _ := strconv.Atoi(string(line[procStart:procEnd])) ent = &LogEntry{ Proc: proc, Start: pos0, } + if faultCall, ok := extractInt(line, "fault-call:"); ok { + ent.Fault = true + ent.FaultCall = faultCall + ent.FaultNth, _ = extractInt(line, "fault-nth:") + } cur = nil continue } @@ -66,3 +68,17 @@ func ParseLog(data []byte) []*LogEntry { } return entries } + +func extractInt(line []byte, prefix string) (int, bool) { + pos := bytes.Index(line, []byte(prefix)) + if pos == -1 { + return 0, false + } + pos += len(prefix) + end := pos + for end != len(line) && line[end] >= '0' && line[end] <= '9' { + end++ + } + v, _ := strconv.Atoi(string(line[pos:end])) + return v, true +} diff --git a/prog/parse_test.go b/prog/parse_test.go index 9492b997e..2f5dd75d6 100644 --- a/prog/parse_test.go +++ b/prog/parse_test.go @@ -25,6 +25,9 @@ gettid() if ent.Proc != 0 { t.Fatalf("proc %v, want 0", ent.Proc) } + if ent.Fault || ent.FaultCall != 0 || ent.FaultNth != 0 { + t.Fatalf("fault injection enabled") + } want := "getpid-gettid" got := ent.P.String() if got != want { @@ -53,6 +56,11 @@ func TestParseMulti(t *testing.T) { entries[4].Proc != 9 { t.Fatalf("bad procs") } + for i, ent := range entries { + if ent.Fault || ent.FaultCall != 0 || ent.FaultNth != 0 { + t.Fatalf("prog %v has fault injection enabled", i) + } + } if s := entries[0].P.String(); s != "getpid-gettid" { t.Fatalf("bad program 0: %s", s) } @@ -89,3 +97,29 @@ getpid() 2015/12/21 12:18:05 executing program 9: munlockall() ` + +func TestParseFault(t *testing.T) { + const execLog = `2015/12/21 12:18:05 executing program 1 (fault-call:1 fault-nth:55): +gettid() +getpid() +` + entries := ParseLog([]byte(execLog)) + if len(entries) != 1 { + t.Fatalf("got %v programs, want 1", len(entries)) + } + ent := entries[0] + if !ent.Fault { + t.Fatalf("fault injection is not enabled") + } + if ent.FaultCall != 1 { + t.Fatalf("fault call: got %v, want 1", ent.FaultCall) + } + if ent.FaultNth != 55 { + t.Fatalf("fault nth: got %v, want 55", ent.FaultNth) + } + want := "gettid-getpid" + got := ent.P.String() + if got != want { + t.Fatalf("bad program: %s, want %s", got, want) + } +} diff --git a/repro/repro.go b/repro/repro.go index 3fbc385c4..b9124942e 100644 --- a/repro/repro.go +++ b/repro/repro.go @@ -171,6 +171,12 @@ func (ctx *context) repro(entries []*prog.LogEntry, crashStart int) (*Result, er var duration time.Duration for _, dur := range []time.Duration{10 * time.Second, 5 * time.Minute} { for _, ent := range suspected { + opts.Fault = ent.Fault + opts.FaultCall = ent.FaultCall + opts.FaultNth = ent.FaultNth + if opts.FaultCall < 0 || opts.FaultCall >= len(ent.P.Calls) { + opts.FaultCall = len(ent.P.Calls) - 1 + } crashed, err := ctx.testProg(ent.P, dur, opts) if err != nil { return nil, err @@ -197,7 +203,11 @@ func (ctx *context) repro(entries []*prog.LogEntry, crashStart int) (*Result, er }() Logf(2, "reproducing crash '%v': minimizing guilty program", ctx.crashDesc) - res.Prog, _ = prog.Minimize(res.Prog, -1, func(p1 *prog.Prog, callIndex int) bool { + call := -1 + if res.Opts.Fault { + call = res.Opts.FaultCall + } + res.Prog, res.Opts.FaultCall = prog.Minimize(res.Prog, call, func(p1 *prog.Prog, callIndex int) bool { crashed, err := ctx.testProg(p1, duration, res.Opts) if err != nil { Logf(1, "reproducing crash '%v': minimization failed with %v", ctx.crashDesc, err) @@ -297,12 +307,15 @@ func (ctx *context) testProg(p *prog.Prog, duration time.Duration, opts csource. return false, fmt.Errorf("failed to copy to VM: %v", err) } - repeat := "1" + repeat := 1 if opts.Repeat { - repeat = "0" + repeat = 0 + } + if !opts.Fault { + opts.FaultCall = -1 } - command := fmt.Sprintf("%v -executor %v -cover=0 -procs=%v -repeat=%v -sandbox %v -threaded=%v -collide=%v %v", - inst.execprogBin, inst.executorBin, opts.Procs, repeat, opts.Sandbox, opts.Threaded, opts.Collide, vmProgFile) + command := fmt.Sprintf("%v -executor %v -cover=0 -procs=%v -repeat=%v -sandbox %v -threaded=%v -collide=%v -fault_call=%v -fault_nth=%v %v", + inst.execprogBin, inst.executorBin, opts.Procs, repeat, opts.Sandbox, opts.Threaded, opts.Collide, opts.FaultCall, opts.FaultNth, vmProgFile) Logf(2, "reproducing crash '%v': testing program (duration=%v, %+v): %s", ctx.crashDesc, duration, opts, p) return ctx.testImpl(inst, command, duration) diff --git a/tools/syz-prog2c/prog2c.go b/tools/syz-prog2c/prog2c.go index f4cc4e3e6..5cdec11f2 100644 --- a/tools/syz-prog2c/prog2c.go +++ b/tools/syz-prog2c/prog2c.go @@ -14,12 +14,14 @@ import ( ) var ( - flagThreaded = flag.Bool("threaded", false, "create threaded program") - flagCollide = flag.Bool("collide", false, "create collide program") - flagRepeat = flag.Bool("repeat", false, "repeat program infinitely or not") - flagProcs = flag.Int("procs", 4, "number of parallel processes") - flagSandbox = flag.String("sandbox", "none", "sandbox to use (none, setuid, namespace)") - flagProg = flag.String("prog", "", "file with program to convert (required)") + flagThreaded = flag.Bool("threaded", false, "create threaded program") + flagCollide = flag.Bool("collide", false, "create collide program") + flagRepeat = flag.Bool("repeat", false, "repeat program infinitely or not") + flagProcs = flag.Int("procs", 4, "number of parallel processes") + flagSandbox = flag.String("sandbox", "none", "sandbox to use (none, setuid, namespace)") + flagProg = flag.String("prog", "", "file with program to convert (required)") + flagFaultCall = flag.Int("fault_call", -1, "inject fault into this call (0-based)") + flagFaultNth = flag.Int("fault_nth", 0, "inject fault on n-th operation (0-based)") ) func main() { @@ -39,12 +41,15 @@ func main() { os.Exit(1) } opts := csource.Options{ - Threaded: *flagThreaded, - Collide: *flagCollide, - Repeat: *flagRepeat, - Procs: *flagProcs, - Sandbox: *flagSandbox, - Repro: false, + Threaded: *flagThreaded, + Collide: *flagCollide, + Repeat: *flagRepeat, + Procs: *flagProcs, + Sandbox: *flagSandbox, + Fault: *flagFaultCall >= 0, + FaultCall: *flagFaultCall, + FaultNth: *flagFaultNth, + Repro: false, } src, err := csource.Write(p, opts) if err != nil { -- cgit mrf-deployment