executor: favor MAP_FIXED_NOREPLACE over MAP_FIXED

MAP_FIXED_NOREPLACE allows to fail early if we happened to overlap with
an existing memory mapping. It should help detects bugs #5674 at an
earlier stage, before it led to memory corruptions.

MAP_FIXED_NOREPLACE is supported from Linux 4.17, which is okay for all
syzkaller use cases on syzbot.

There's no such option for some of the supported OSes, so set it
depending on the configuration we're building for.

4 files changed, 13 insertions, 5 deletions

diff --git a/executor/executor.cc b/executor/executor.cc
index 702f8c3b4..4ea40e0f2 100644
--- a/executor/executor.cc
+++ b/executor/executor.cc
@@ -484,6 +484,14 @@ static void parse_handshake(const handshake_req& req);
 #include "syscalls.h"
 
 #if GOOS_linux
+#define MAP_FIXED_EXCLUSIVE MAP_FIXED_NOREPLACE
+#elif GOOS_freebsd
+#define MAP_FIXED_EXCLUSIVE (MAP_FIXED | MAP_EXCL)
+#else
+#define MAP_FIXED_EXCLUSIVE MAP_FIXED // The check is not supported.
+#endif
+
+#if GOOS_linux
 #include "executor_linux.h"
 #elif GOOS_fuchsia
 #include "executor_fuchsia.h"
diff --git a/executor/executor_bsd.h b/executor/executor_bsd.h
index e8530f614..0eb76c588 100644
--- a/executor/executor_bsd.h
+++ b/executor/executor_bsd.h
@@ -27,7 +27,7 @@ static void os_init(int argc, char** argv, void* data, size_t data_size)
 	int prot = PROT_READ | PROT_WRITE | PROT_EXEC;
 #endif
 
-	int flags = MAP_ANON | MAP_PRIVATE | MAP_FIXED;
+	int flags = MAP_ANON | MAP_PRIVATE | MAP_FIXED_EXCLUSIVE;
 #if GOOS_freebsd
 	// Fail closed if the chosen data offset conflicts with an existing mapping.
 	flags |= MAP_EXCL;
diff --git a/executor/executor_linux.h b/executor/executor_linux.h
index e952e6ea9..9784700ba 100644
--- a/executor/executor_linux.h
+++ b/executor/executor_linux.h
@@ -59,13 +59,13 @@ static void os_init(int argc, char** argv, char* data, size_t data_size)
 	// One observed case before: executor had a mapping above the data mapping (output region),
 	// while C repros did not have that mapping above, as the result in one case VMA had next link,
 	// while in the other it didn't and it caused a bug to not reproduce with the C repro.
-	void* got = mmap(data - SYZ_PAGE_SIZE, SYZ_PAGE_SIZE, PROT_NONE, MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
+	void* got = mmap(data - SYZ_PAGE_SIZE, SYZ_PAGE_SIZE, PROT_NONE, MAP_ANON | MAP_PRIVATE | MAP_FIXED_EXCLUSIVE, -1, 0);
 	if (data - SYZ_PAGE_SIZE != got)
 		failmsg("mmap of left data PROT_NONE page failed", "want %p, got %p", data - SYZ_PAGE_SIZE, got);
-	got = mmap(data, data_size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
+	got = mmap(data, data_size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_ANON | MAP_PRIVATE | MAP_FIXED_EXCLUSIVE, -1, 0);
 	if (data != got)
 		failmsg("mmap of data segment failed", "want %p, got %p", data, got);
-	got = mmap(data + data_size, SYZ_PAGE_SIZE, PROT_NONE, MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
+	got = mmap(data + data_size, SYZ_PAGE_SIZE, PROT_NONE, MAP_ANON | MAP_PRIVATE | MAP_FIXED_EXCLUSIVE, -1, 0);
 	if (data + data_size != got)
 		failmsg("mmap of right data PROT_NONE page failed", "want %p, got %p", data + data_size, got);
 
diff --git a/executor/executor_test.h b/executor/executor_test.h
index c94de09a2..e2a2002bb 100644
--- a/executor/executor_test.h
+++ b/executor/executor_test.h
@@ -22,7 +22,7 @@ static void os_init(int argc, char** argv, void* data, size_t data_size)
 	if (getppid() == 1)
 		exitf("the parent process was killed");
 #endif
-	void* got = mmap(data, data_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE | MAP_FIXED, -1, 0);
+	void* got = mmap(data, data_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE | MAP_FIXED_EXCLUSIVE, -1, 0);
 	if (data != got)
 		failmsg("mmap of data segment failed", "want %p, got %p", data, got);
 	is_kernel_64_bit = sizeof(unsigned long) == 8;