executor: implement support for leak checking

Leak checking support was half done and did not really work.
This is heavy-lifting to make it work.

1. Move leak/fault setup into executor.
pkg/host was a wrong place for them because we need then in C repros too.
The pkg/host periodic callback functionality did not work too,
we need it in executor so that we can reuse it in C repros too.
Remove setup/callback functions in pkg/host entirely.

2. Do leak setup/checking in C repros.
The way leak checking is invoked is slightly different from fuzzer,
but much better then no support at all.
At least the checking code is shared.

3. Add Leak option to pkg/csource and -leak flag to syz-prog2c.

4. Don't enalbe leak checking in fuzzer while we are triaging initial corpus.
It's toooo slow.

5. Fix pkg/repro to do something more sane for leak bugs.

Few other minor fixes here and there.

4 files changed, 223 insertions, 41 deletions

diff --git a/executor/common.h b/executor/common.h
index 27a7380f7..db2e1204e 100644
--- a/executor/common.h
+++ b/executor/common.h
@@ -145,7 +145,8 @@ static void sleep_ms(uint64 ms)
 }
 #endif
 
-#if SYZ_EXECUTOR || SYZ_THREADED || SYZ_REPEAT && SYZ_EXECUTOR_USES_FORK_SERVER
+#if SYZ_EXECUTOR || SYZ_THREADED || SYZ_REPEAT && SYZ_EXECUTOR_USES_FORK_SERVER || \
+    SYZ_ENABLE_LEAK
 #include <time.h>
 
 static uint64 current_time_ms(void)
@@ -218,7 +219,7 @@ static void remove_dir(const char* dir)
 #endif
 
 #if !GOOS_linux
-#if SYZ_EXECUTOR || SYZ_FAULT_INJECTION
+#if SYZ_EXECUTOR
 static int inject_fault(int nth)
 {
 	return 0;
@@ -638,6 +639,11 @@ static void loop(void)
 #if SYZ_EXECUTOR || SYZ_USE_TMP_DIR
 		remove_dir(cwdbuf);
 #endif
+#if SYZ_ENABLE_LEAK
+		// Note: this will fail under setuid sandbox because we don't have
+		// write permissions for the kmemleak file.
+		check_leaks();
+#endif
 	}
 }
 #else
@@ -686,6 +692,16 @@ int main(void)
 	/*MMAP_DATA*/
 #endif
 
+#if SYZ_ENABLE_BINFMT_MISC
+	setup_binfmt_misc();
+#endif
+#if SYZ_ENABLE_LEAK
+	setup_leak();
+#endif
+#if SYZ_FAULT_INJECTION
+	setup_fault();
+#endif
+
 #if SYZ_HANDLE_SEGV
 	install_segv_handler();
 #endif
@@ -706,6 +722,9 @@ int main(void)
 	}
 	sleep(1000000);
 #endif
+#if !SYZ_PROCS && !SYZ_REPEAT && SYZ_ENABLE_LEAK
+	check_leaks();
+#endif
 	return 0;
 }
 #endif
diff --git a/executor/common_linux.h b/executor/common_linux.h
index 2d1460ee6..4c33acdea 100644
--- a/executor/common_linux.h
+++ b/executor/common_linux.h
@@ -71,7 +71,8 @@ static int event_timedwait(event_t* ev, uint64 timeout)
 #endif
 
 #if SYZ_EXECUTOR || SYZ_REPEAT || SYZ_TUN_ENABLE || SYZ_FAULT_INJECTION || SYZ_SANDBOX_NONE || \
-    SYZ_SANDBOX_SETUID || SYZ_SANDBOX_NAMESPACE || SYZ_SANDBOX_ANDROID_UNTRUSTED_APP
+    SYZ_SANDBOX_SETUID || SYZ_SANDBOX_NAMESPACE || SYZ_SANDBOX_ANDROID_UNTRUSTED_APP ||        \
+    SYZ_FAULT_INJECTION || SYZ_ENABLE_LEAK || SYZ_ENABLE_BINFMT_MISC
 #include <errno.h>
 #include <fcntl.h>
 #include <stdarg.h>
@@ -1868,26 +1869,6 @@ void initialize_cgroups()
 #endif
 #endif
 
-#if SYZ_EXECUTOR || (SYZ_ENABLE_BINFMT_MISC && (SYZ_SANDBOX_NONE || SYZ_SANDBOX_SETUID || SYZ_SANDBOX_NAMESPACE || SYZ_SANDBOX_ANDROID_UNTRUSTED_APP))
-#include <fcntl.h>
-#include <sys/mount.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-
-static void setup_binfmt_misc()
-{
-#if SYZ_EXECUTOR
-	if (!flag_enable_binfmt_misc)
-		return;
-#endif
-	if (mount(0, "/proc/sys/fs/binfmt_misc", "binfmt_misc", 0, 0)) {
-		debug("mount(binfmt_misc) failed: %d\n", errno);
-	}
-	write_file("/proc/sys/fs/binfmt_misc/register", ":syz0:M:0:\x01::./file0:");
-	write_file("/proc/sys/fs/binfmt_misc/register", ":syz1:M:1:\x02::./file0:POC");
-}
-#endif
-
 #if SYZ_EXECUTOR || SYZ_SANDBOX_NONE || SYZ_SANDBOX_SETUID || SYZ_SANDBOX_NAMESPACE || SYZ_SANDBOX_ANDROID_UNTRUSTED_APP
 #include <errno.h>
 #include <sys/mount.h>
@@ -1900,9 +1881,6 @@ static void setup_common()
 #if SYZ_EXECUTOR || SYZ_ENABLE_CGROUPS
 	setup_cgroups();
 #endif
-#if SYZ_EXECUTOR || SYZ_ENABLE_BINFMT_MISC
-	setup_binfmt_misc();
-#endif
 }
 
 #include <sched.h>
@@ -2475,10 +2453,6 @@ retry:
 
 static int inject_fault(int nth)
 {
-#if SYZ_EXECUTOR
-	if (!flag_enable_fault_injection)
-		return 0;
-#endif
 	int fd;
 	fd = open("/proc/thread-self/fail-nth", O_RDWR);
 	// We treat errors here as temporal/non-critical because we see
@@ -2497,8 +2471,6 @@ static int inject_fault(int nth)
 #if SYZ_EXECUTOR
 static int fault_injected(int fail_fd)
 {
-	if (!flag_enable_fault_injection)
-		return 0;
 	char buf[16];
 	int n = read(fail_fd, buf, sizeof(buf) - 1);
 	if (n <= 0)
@@ -2646,3 +2618,153 @@ static void close_fds()
 		close(fd);
 }
 #endif
+
+#if SYZ_EXECUTOR || SYZ_FAULT_INJECTION
+#include <errno.h>
+
+static void setup_fault()
+{
+	static struct {
+		const char* file;
+		const char* val;
+		bool fatal;
+	} files[] = {
+	    {"/sys/kernel/debug/failslab/ignore-gfp-wait", "N", true},
+	    // These are enabled by separate configs (e.g. CONFIG_FAIL_FUTEX)
+	    // and we did not check all of them in host.checkFaultInjection, so we ignore errors.
+	    {"/sys/kernel/debug/fail_futex/ignore-private", "N", false},
+	    {"/sys/kernel/debug/fail_page_alloc/ignore-gfp-highmem", "N", false},
+	    {"/sys/kernel/debug/fail_page_alloc/ignore-gfp-wait", "N", false},
+	    {"/sys/kernel/debug/fail_page_alloc/min-order", "0", false},
+	};
+	unsigned i;
+	for (i = 0; i < sizeof(files) / sizeof(files[0]); i++) {
+		if (!write_file(files[i].file, files[i].val)) {
+			debug("failed to write %s: %d\n", files[i].file, errno);
+			if (files[i].fatal)
+				fail("failed to write %s", files[i].file);
+		}
+	}
+}
+#endif
+
+#if SYZ_EXECUTOR || SYZ_ENABLE_LEAK
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#define KMEMLEAK_FILE "/sys/kernel/debug/kmemleak"
+
+static void setup_leak()
+{
+	// Flush boot leaks.
+	if (!write_file(KMEMLEAK_FILE, "scan"))
+		fail("failed to write %s", KMEMLEAK_FILE);
+	sleep(5); // account for MSECS_MIN_AGE
+	if (!write_file(KMEMLEAK_FILE, "scan"))
+		fail("failed to write %s", KMEMLEAK_FILE);
+	if (!write_file(KMEMLEAK_FILE, "clear"))
+		fail("failed to write %s", KMEMLEAK_FILE);
+}
+
+#define SYZ_HAVE_LEAK_CHECK 1
+#if SYZ_EXECUTOR
+static void check_leaks(char** frames, int nframes)
+#else
+static void check_leaks(void)
+#endif
+{
+	int fd = open(KMEMLEAK_FILE, O_RDWR);
+	if (fd == -1)
+		fail("failed to open(\"%s\")", KMEMLEAK_FILE);
+	// KMEMLEAK has false positives. To mitigate most of them, it checksums
+	// potentially leaked objects, and reports them only on the next scan
+	// iff the checksum does not change. Because of that we do the following
+	// intricate dance:
+	// Scan, sleep, scan again. At this point we can get some leaks.
+	// If there are leaks, we sleep and scan again, this can remove
+	// false leaks. Then, read kmemleak again. If we get leaks now, then
+	// hopefully these are true positives during the previous testing cycle.
+	uint64 start = current_time_ms();
+	if (write(fd, "scan", 4) != 4)
+		fail("failed to write(%s, \"scan\")", KMEMLEAK_FILE);
+	sleep(1);
+	// Account for MSECS_MIN_AGE
+	// (1 second less because scanning will take at least a second).
+	while (current_time_ms() - start < 4 * 1000)
+		sleep(1);
+	if (write(fd, "scan", 4) != 4)
+		fail("failed to write(%s, \"scan\")", KMEMLEAK_FILE);
+	static char buf[128 << 10];
+	ssize_t n = read(fd, buf, sizeof(buf) - 1);
+	if (n < 0)
+		fail("failed to read(%s)", KMEMLEAK_FILE);
+#if SYZ_EXECUTOR
+	int nleaks = 0;
+#endif
+	if (n != 0) {
+		sleep(1);
+		if (write(fd, "scan", 4) != 4)
+			fail("failed to write(%s, \"scan\")", KMEMLEAK_FILE);
+		if (lseek(fd, 0, SEEK_SET) < 0)
+			fail("failed to lseek(%s)", KMEMLEAK_FILE);
+		n = read(fd, buf, sizeof(buf) - 1);
+		if (n < 0)
+			fail("failed to read(%s)", KMEMLEAK_FILE);
+		buf[n] = 0;
+		char* pos = buf;
+		char* end = buf + n;
+		while (pos < end) {
+			char* next = strstr(pos + 1, "unreferenced object");
+			if (!next)
+				next = end;
+			char prev = *next;
+			*next = 0;
+#if SYZ_EXECUTOR
+			int f;
+			for (f = 0; f < nframes; f++) {
+				if (strstr(pos, frames[f]))
+					break;
+			}
+			if (f != nframes) {
+				*next = prev;
+				pos = next;
+				continue;
+			}
+#endif
+			// BUG in output should be recognized by manager.
+			fprintf(stderr, "BUG: memory leak\n%s\n", pos);
+			*next = prev;
+			pos = next;
+#if SYZ_EXECUTOR
+			nleaks++;
+#endif
+		}
+	}
+	if (write(fd, "clear", 5) != 5)
+		fail("failed to write(%s, \"clear\")", KMEMLEAK_FILE);
+	close(fd);
+#if SYZ_EXECUTOR
+	if (nleaks)
+		doexit(1);
+#endif
+}
+#endif
+
+#if SYZ_EXECUTOR || SYZ_ENABLE_BINFMT_MISC
+#include <fcntl.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+static void setup_binfmt_misc()
+{
+	if (mount(0, "/proc/sys/fs/binfmt_misc", "binfmt_misc", 0, 0)) {
+		debug("mount(binfmt_misc) failed: %d\n", errno);
+	}
+	write_file("/proc/sys/fs/binfmt_misc/register", ":syz0:M:0:\x01::./file0:");
+	write_file("/proc/sys/fs/binfmt_misc/register", ":syz1:M:1:\x02::./file0:POC");
+}
+#endif
diff --git a/executor/executor.cc b/executor/executor.cc
index 50b922182..4dfe9490f 100644
--- a/executor/executor.cc
+++ b/executor/executor.cc
@@ -113,12 +113,10 @@ static bool flag_debug;
 static bool flag_cover;
 static sandbox_type flag_sandbox;
 static bool flag_extra_cover;
-static bool flag_enable_fault_injection;
 static bool flag_enable_tun;
 static bool flag_enable_net_dev;
 static bool flag_enable_net_reset;
 static bool flag_enable_cgroups;
-static bool flag_enable_binfmt_misc;
 static bool flag_enable_close_fds;
 
 static bool flag_collect_cover;
@@ -287,6 +285,11 @@ struct kcov_comparison_t {
 	bool operator<(const struct kcov_comparison_t& other) const;
 };
 
+struct feature_t {
+	const char* name;
+	void (*setup)();
+};
+
 static thread_t* schedule_call(int call_index, int call_num, bool colliding, uint64 copyout_index, uint64 num_args, uint64* args, uint64* pos);
 static void handle_completion(thread_t* th);
 static void copyout_call_results(thread_t* th);
@@ -303,6 +306,7 @@ static uint64 swap(uint64 v, uint64 size, uint64 bf);
 static void copyin(char* addr, uint64 val, uint64 size, uint64 bf, uint64 bf_off, uint64 bf_len);
 static bool copyout(char* addr, uint64 size, uint64* res);
 static void setup_control_pipes();
+static void setup_features(char** enable, int n);
 
 #include "syscalls.h"
 
@@ -330,6 +334,18 @@ int main(int argc, char** argv)
 		puts(GOOS " " GOARCH " " SYZ_REVISION " " GIT_REVISION);
 		return 0;
 	}
+	if (argc >= 2 && strcmp(argv[1], "setup") == 0) {
+		setup_features(argv + 2, argc - 2);
+		return 0;
+	}
+	if (argc >= 2 && strcmp(argv[1], "leak") == 0) {
+#if SYZ_HAVE_LEAK_CHECK
+		check_leaks(argv + 2, argc - 2);
+#else
+		fail("leak checking is not implemented");
+#endif
+		return 0;
+	}
 	if (argc == 2 && strcmp(argv[1], "test") == 0)
 		return run_tests();
 
@@ -449,13 +465,11 @@ void parse_env_flags(uint64 flags)
 	else if (flags & (1 << 4))
 		flag_sandbox = sandbox_android_untrusted_app;
 	flag_extra_cover = flags & (1 << 5);
-	flag_enable_fault_injection = flags & (1 << 6);
-	flag_enable_tun = flags & (1 << 7);
-	flag_enable_net_dev = flags & (1 << 8);
-	flag_enable_net_reset = flags & (1 << 9);
-	flag_enable_cgroups = flags & (1 << 10);
-	flag_enable_binfmt_misc = flags & (1 << 11);
-	flag_enable_close_fds = flags & (1 << 12);
+	flag_enable_tun = flags & (1 << 6);
+	flag_enable_net_dev = flags & (1 << 7);
+	flag_enable_net_reset = flags & (1 << 8);
+	flag_enable_cgroups = flags & (1 << 9);
+	flag_enable_close_fds = flags & (1 << 10);
 }
 
 #if SYZ_EXECUTOR_USES_FORK_SERVER
@@ -1359,6 +1373,26 @@ bool kcov_comparison_t::operator<(const struct kcov_comparison_t& other) const
 }
 #endif
 
+void setup_features(char** enable, int n)
+{
+	// This does any one-time setup for the requested features on the machine.
+	// Note: this can be called multiple times and must be idempotent.
+	for (int i = 0; i < n; i++) {
+		bool found = false;
+#if SYZ_HAVE_FEATURES
+		for (unsigned f = 0; f < sizeof(features) / sizeof(features[0]); f++) {
+			if (strcmp(enable[i], features[f].name) == 0) {
+				features[f].setup();
+				found = true;
+				break;
+			}
+		}
+#endif
+		if (!found)
+			fail("unknown feature %s", enable[i]);
+	}
+}
+
 void fail(const char* msg, ...)
 {
 	int e = errno;
diff --git a/executor/executor_linux.h b/executor/executor_linux.h
index f98a93f90..867f1ec4d 100644
--- a/executor/executor_linux.h
+++ b/executor/executor_linux.h
@@ -167,3 +167,10 @@ NORETURN void doexit(int status)
 	for (i = 0;; i++) {
 	}
 }
+
+#define SYZ_HAVE_FEATURES 1
+static feature_t features[] = {
+    {"leak", setup_leak},
+    {"fault", setup_fault},
+    {"binfmt_misc", setup_binfmt_misc},
+};