executor: sandbox with memory/pid cgroups

Set limit of 32 pids and 200MB per test process.
This should prevent things like fork bombs and frequent OOMs.

Fixes #589

1 files changed, 42 insertions, 10 deletions

diff --git a/executor/common_linux.h b/executor/common_linux.h
index 40b06f9fe..21433c515 100644
--- a/executor/common_linux.h
+++ b/executor/common_linux.h
@@ -1416,6 +1416,10 @@ static void setup_cgroups()
 	if (chmod("/syzcgroup/net", 0777)) {
 		debug("chmod(/syzcgroup/net) failed: %d\n", errno);
 	}
+	// We are going to setup memory limits and we want the test process to be killed.
+	if (!write_file("/proc/self/oom_score_adj", "-1000")) {
+		debug("write(oom_score_adj) failed: %d\n", errno);
+	}
 }
 
 // TODO(dvyukov): this should be under a separate define for separate minimization,
@@ -1962,30 +1966,58 @@ static void setup_loop()
 #if SYZ_EXECUTOR || SYZ_ENABLE_CGROUPS
 	int pid = getpid();
 	char cgroupdir[64];
-	char procs_file[128];
+	char file[128];
 	snprintf(cgroupdir, sizeof(cgroupdir), "/syzcgroup/unified/syz%llu", procid);
 	if (mkdir(cgroupdir, 0777)) {
 		debug("mkdir(%s) failed: %d\n", cgroupdir, errno);
 	}
-	snprintf(procs_file, sizeof(procs_file), "%s/cgroup.procs", cgroupdir);
-	if (!write_file(procs_file, "%d", pid)) {
-		debug("write(%s) failed: %d\n", procs_file, errno);
+	// Restrict number of pids per test process to prevent fork bombs.
+	// We have up to 16 threads + main process + loop.
+	// 32 pids should be enough for everyone.
+	snprintf(file, sizeof(file), "%s/pids.max", cgroupdir);
+	if (!write_file(file, "32")) {
+		debug("write(%s) failed: %d\n", file, errno);
+	}
+	// Restrict memory consumption.
+	// We have some syscalls that inherently consume lots of memory,
+	// e.g. mounting some filesystem images requires at least 128MB
+	// image in memory. We restrict RLIMIT_AS to 160MB. Here we gradually
+	// increase low/high/max limits to make things more interesting.
+	snprintf(file, sizeof(file), "%s/memory.low", cgroupdir);
+	if (!write_file(file, "%d", 198 << 20)) {
+		debug("write(%s) failed: %d\n", file, errno);
+	}
+	snprintf(file, sizeof(file), "%s/memory.high", cgroupdir);
+	if (!write_file(file, "%d", 199 << 20)) {
+		debug("write(%s) failed: %d\n", file, errno);
+	}
+	snprintf(file, sizeof(file), "%s/memory.max", cgroupdir);
+	if (!write_file(file, "%d", 200 << 20)) {
+		debug("write(%s) failed: %d\n", file, errno);
+	}
+	if (!write_file("/proc/self/oom_score_adj", "-1000")) {
+		debug("write(oom_score_adj) failed: %d\n", errno);
+	}
+	// Setup some v1 groups to make things more interesting.
+	snprintf(file, sizeof(file), "%s/cgroup.procs", cgroupdir);
+	if (!write_file(file, "%d", pid)) {
+		debug("write(%s) failed: %d\n", file, errno);
 	}
 	snprintf(cgroupdir, sizeof(cgroupdir), "/syzcgroup/cpu/syz%llu", procid);
 	if (mkdir(cgroupdir, 0777)) {
 		debug("mkdir(%s) failed: %d\n", cgroupdir, errno);
 	}
-	snprintf(procs_file, sizeof(procs_file), "%s/cgroup.procs", cgroupdir);
-	if (!write_file(procs_file, "%d", pid)) {
-		debug("write(%s) failed: %d\n", procs_file, errno);
+	snprintf(file, sizeof(file), "%s/cgroup.procs", cgroupdir);
+	if (!write_file(file, "%d", pid)) {
+		debug("write(%s) failed: %d\n", file, errno);
 	}
 	snprintf(cgroupdir, sizeof(cgroupdir), "/syzcgroup/net/syz%llu", procid);
 	if (mkdir(cgroupdir, 0777)) {
 		debug("mkdir(%s) failed: %d\n", cgroupdir, errno);
 	}
-	snprintf(procs_file, sizeof(procs_file), "%s/cgroup.procs", cgroupdir);
-	if (!write_file(procs_file, "%d", pid)) {
-		debug("write(%s) failed: %d\n", procs_file, errno);
+	snprintf(file, sizeof(file), "%s/cgroup.procs", cgroupdir);
+	if (!write_file(file, "%d", pid)) {
+		debug("write(%s) failed: %d\n", file, errno);
 	}
 #endif
 #if SYZ_EXECUTOR || SYZ_RESET_NET_NAMESPACE