pkg/flatrpc, pkg/vminfo, executor: introduce readonly coverage

Add a new vminfo feature, FeatureKcovResetIoctl, that is true if the
kernel supports ioctl(KCOV_RESET_TRACE) making it possible to reset the
coverage buffer on the kernel side. This, in turn, allows us to map the
coverage buffer read-only, which will prevent all sorts of
userspace-generated corruptions at a cost of an extra syscall per program
execution.

The corresponding exec env flag, ExecEnv::ReadOnlyCoverage, turns on
read-only coverage in the executor. It is enabled by default
if FeatureKcovResetIoctl is on.

7 files changed, 135 insertions, 75 deletions

diff --git a/executor/executor.cc b/executor/executor.cc
index a262bff83..25fba22e7 100644
--- a/executor/executor.cc
+++ b/executor/executor.cc
@@ -257,6 +257,7 @@ static uint64 start_time_ms = 0;
 static bool flag_debug;
 static bool flag_snapshot;
 static bool flag_coverage;
+static bool flag_read_only_coverage;
 static bool flag_sandbox_none;
 static bool flag_sandbox_setuid;
 static bool flag_sandbox_namespace;
@@ -777,6 +778,7 @@ void parse_handshake(const handshake_req& req)
 	slowdown_scale = req.slowdown_scale;
 	flag_debug = (bool)(req.flags & rpc::ExecEnv::Debug);
 	flag_coverage = (bool)(req.flags & rpc::ExecEnv::Signal);
+	flag_read_only_coverage = (bool)(req.flags & rpc::ExecEnv::ReadOnlyCoverage);
 	flag_sandbox_none = (bool)(req.flags & rpc::ExecEnv::SandboxNone);
 	flag_sandbox_setuid = (bool)(req.flags & rpc::ExecEnv::SandboxSetuid);
 	flag_sandbox_namespace = (bool)(req.flags & rpc::ExecEnv::SandboxNamespace);
diff --git a/executor/executor_linux.h b/executor/executor_linux.h
index 0ae1155bb..f882b9c40 100644
--- a/executor/executor_linux.h
+++ b/executor/executor_linux.h
@@ -37,6 +37,7 @@ struct kcov_remote_arg {
 #define KCOV_ENABLE _IO('c', 100)
 #define KCOV_DISABLE _IO('c', 101)
 #define KCOV_REMOTE_ENABLE _IOW('c', 102, kcov_remote_arg<0>)
+#define KCOV_RESET_TRACE _IO('c', 104)
 
 #define KCOV_SUBSYSTEM_COMMON (0x00ull << 56)
 #define KCOV_SUBSYSTEM_USB (0x01ull << 56)
@@ -142,17 +143,27 @@ static void cover_mmap(cover_t* cov)
 	if (mapped == MAP_FAILED)
 		exitf("failed to preallocate kcov buffer");
 	// Now map the kcov buffer to the file, overwriting the existing mapping above.
+	int prot = flag_read_only_coverage ? PROT_READ : (PROT_READ | PROT_WRITE);
 	cov->data = (char*)mmap(mapped + SYZ_PAGE_SIZE, cov->mmap_alloc_size,
-				PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, cov->fd, 0);
+				prot, MAP_SHARED | MAP_FIXED, cov->fd, 0);
 	if (cov->data == MAP_FAILED)
 		exitf("cover mmap failed");
-	if (pkeys_enabled && pkey_mprotect(cov->data, cov->mmap_alloc_size, PROT_READ | PROT_WRITE, RESERVED_PKEY))
+	if (pkeys_enabled && pkey_mprotect(cov->data, cov->mmap_alloc_size, prot, RESERVED_PKEY))
 		exitf("failed to pkey_mprotect kcov buffer");
 	cov->data_end = cov->data + cov->mmap_alloc_size;
 	cov->data_offset = is_kernel_64_bit ? sizeof(uint64_t) : sizeof(uint32_t);
 	cov->pc_offset = 0;
 }
 
+static void cover_munmap(cover_t* cov)
+{
+	if (cov->data == NULL)
+		fail("cover_munmap invoked on a non-mmapped cover_t object");
+	if (munmap(cov->data - SYZ_PAGE_SIZE, cov->mmap_alloc_size + 2 * SYZ_PAGE_SIZE))
+		fail("cover_munmap failed");
+	cov->data = NULL;
+}
+
 static void cover_enable(cover_t* cov, bool collect_comps, bool extra)
 {
 	unsigned int kcov_mode = collect_comps ? KCOV_TRACE_CMP : KCOV_TRACE_PC;
@@ -186,9 +197,14 @@ static void cover_reset(cover_t* cov)
 			fail("cover_reset: current_thread == 0");
 		cov = &current_thread->cov;
 	}
-	cover_unprotect(cov);
-	*(uint64*)cov->data = 0;
-	cover_protect(cov);
+	if (flag_read_only_coverage) {
+		if (ioctl(cov->fd, KCOV_RESET_TRACE, 0))
+			fail("KCOV_RESET_TRACE failed");
+	} else {
+		cover_unprotect(cov);
+		*(uint64*)cov->data = 0;
+		cover_protect(cov);
+	}
 	cov->overflow = false;
 }
 
@@ -307,9 +323,34 @@ static const char* setup_delay_kcov()
 	return error;
 }
 
+static const char* setup_kcov_reset_ioctl()
+{
+	int fd = open("/sys/kernel/debug/kcov", O_RDWR);
+	if (fd == -1)
+		return "open of /sys/kernel/debug/kcov failed";
+	cover_t cov = {};
+	cov.fd = kCoverFd;
+	cover_open(&cov, false);
+	cover_mmap(&cov);
+	const char* error = NULL;
+	cover_enable(&cov, false, false);
+	int ret;
+	if ((ret = ioctl(cov.fd, KCOV_RESET_TRACE, 0))) {
+		if (errno != ENOTTY) {
+			fprintf(stderr, "ret: %d, errno: %d\n", ret, errno);
+			fail("ioctl(KCOV_RESET_TRACE) failed");
+		}
+		error = "kernel does not support ioctl(KCOV_RESET_TRACE)";
+	}
+	cover_munmap(&cov);
+	close(cov.fd);
+	return error;
+}
+
 #define SYZ_HAVE_FEATURES 1
 static feature_t features[] = {
     {rpc::Feature::DelayKcovMmap, setup_delay_kcov},
+    {rpc::Feature::KcovResetIoctl, setup_kcov_reset_ioctl},
     {rpc::Feature::Fault, setup_fault},
     {rpc::Feature::Leak, setup_leak},
     {rpc::Feature::KCSAN, setup_kcsan},
diff --git a/pkg/flatrpc/flatrpc.fbs b/pkg/flatrpc/flatrpc.fbs
index 58dc7b292..3876af965 100644
--- a/pkg/flatrpc/flatrpc.fbs
+++ b/pkg/flatrpc/flatrpc.fbs
@@ -16,6 +16,7 @@ enum Feature : uint64 (bit_flags) {
 	Comparisons,
 	ExtraCoverage,
 	DelayKcovMmap,
+	KcovResetIoctl,
 	SandboxNone,
 	SandboxSetuid,
 	SandboxNamespace,
@@ -135,6 +136,7 @@ enum RequestFlag : uint64 (bit_flags) {
 enum ExecEnv : uint64 (bit_flags) {
 	Debug,			// debug output from executor
 	Signal,			// collect feedback signals (coverage)
+	ReadOnlyCoverage,	// map coverage as readonly, use an ioctl to reset it
 	ResetState,    		// fully reset executor state befor executing the test
 	SandboxNone,		// minimal sandboxing
 	SandboxSetuid,		// impersonate nobody user
diff --git a/pkg/flatrpc/flatrpc.go b/pkg/flatrpc/flatrpc.go
index d04d5c531..c7afd2c77 100644
--- a/pkg/flatrpc/flatrpc.go
+++ b/pkg/flatrpc/flatrpc.go
@@ -45,23 +45,24 @@ const (
 	FeatureComparisons      Feature = 2
 	FeatureExtraCoverage    Feature = 4
 	FeatureDelayKcovMmap    Feature = 8
-	FeatureSandboxNone      Feature = 16
-	FeatureSandboxSetuid    Feature = 32
-	FeatureSandboxNamespace Feature = 64
-	FeatureSandboxAndroid   Feature = 128
-	FeatureFault            Feature = 256
-	FeatureLeak             Feature = 512
-	FeatureNetInjection     Feature = 1024
-	FeatureNetDevices       Feature = 2048
-	FeatureKCSAN            Feature = 4096
-	FeatureDevlinkPCI       Feature = 8192
-	FeatureNicVF            Feature = 16384
-	FeatureUSBEmulation     Feature = 32768
-	FeatureVhciInjection    Feature = 65536
-	FeatureWifiEmulation    Feature = 131072
-	FeatureLRWPANEmulation  Feature = 262144
-	FeatureBinFmtMisc       Feature = 524288
-	FeatureSwap             Feature = 1048576
+	FeatureKcovResetIoctl   Feature = 16
+	FeatureSandboxNone      Feature = 32
+	FeatureSandboxSetuid    Feature = 64
+	FeatureSandboxNamespace Feature = 128
+	FeatureSandboxAndroid   Feature = 256
+	FeatureFault            Feature = 512
+	FeatureLeak             Feature = 1024
+	FeatureNetInjection     Feature = 2048
+	FeatureNetDevices       Feature = 4096
+	FeatureKCSAN            Feature = 8192
+	FeatureDevlinkPCI       Feature = 16384
+	FeatureNicVF            Feature = 32768
+	FeatureUSBEmulation     Feature = 65536
+	FeatureVhciInjection    Feature = 131072
+	FeatureWifiEmulation    Feature = 262144
+	FeatureLRWPANEmulation  Feature = 524288
+	FeatureBinFmtMisc       Feature = 1048576
+	FeatureSwap             Feature = 2097152
 )
 
 var EnumNamesFeature = map[Feature]string{
@@ -69,6 +70,7 @@ var EnumNamesFeature = map[Feature]string{
 	FeatureComparisons:      "Comparisons",
 	FeatureExtraCoverage:    "ExtraCoverage",
 	FeatureDelayKcovMmap:    "DelayKcovMmap",
+	FeatureKcovResetIoctl:   "KcovResetIoctl",
 	FeatureSandboxNone:      "SandboxNone",
 	FeatureSandboxSetuid:    "SandboxSetuid",
 	FeatureSandboxNamespace: "SandboxNamespace",
@@ -93,6 +95,7 @@ var EnumValuesFeature = map[string]Feature{
 	"Comparisons":      FeatureComparisons,
 	"ExtraCoverage":    FeatureExtraCoverage,
 	"DelayKcovMmap":    FeatureDelayKcovMmap,
+	"KcovResetIoctl":   FeatureKcovResetIoctl,
 	"SandboxNone":      FeatureSandboxNone,
 	"SandboxSetuid":    FeatureSandboxSetuid,
 	"SandboxNamespace": FeatureSandboxNamespace,
@@ -313,27 +316,29 @@ type ExecEnv uint64
 const (
 	ExecEnvDebug               ExecEnv = 1
 	ExecEnvSignal              ExecEnv = 2
-	ExecEnvResetState          ExecEnv = 4
-	ExecEnvSandboxNone         ExecEnv = 8
-	ExecEnvSandboxSetuid       ExecEnv = 16
-	ExecEnvSandboxNamespace    ExecEnv = 32
-	ExecEnvSandboxAndroid      ExecEnv = 64
-	ExecEnvExtraCover          ExecEnv = 128
-	ExecEnvEnableTun           ExecEnv = 256
-	ExecEnvEnableNetDev        ExecEnv = 512
-	ExecEnvEnableNetReset      ExecEnv = 1024
-	ExecEnvEnableCgroups       ExecEnv = 2048
-	ExecEnvEnableCloseFds      ExecEnv = 4096
-	ExecEnvEnableDevlinkPCI    ExecEnv = 8192
-	ExecEnvEnableVhciInjection ExecEnv = 16384
-	ExecEnvEnableWifi          ExecEnv = 32768
-	ExecEnvDelayKcovMmap       ExecEnv = 65536
-	ExecEnvEnableNicVF         ExecEnv = 131072
+	ExecEnvReadOnlyCoverage    ExecEnv = 4
+	ExecEnvResetState          ExecEnv = 8
+	ExecEnvSandboxNone         ExecEnv = 16
+	ExecEnvSandboxSetuid       ExecEnv = 32
+	ExecEnvSandboxNamespace    ExecEnv = 64
+	ExecEnvSandboxAndroid      ExecEnv = 128
+	ExecEnvExtraCover          ExecEnv = 256
+	ExecEnvEnableTun           ExecEnv = 512
+	ExecEnvEnableNetDev        ExecEnv = 1024
+	ExecEnvEnableNetReset      ExecEnv = 2048
+	ExecEnvEnableCgroups       ExecEnv = 4096
+	ExecEnvEnableCloseFds      ExecEnv = 8192
+	ExecEnvEnableDevlinkPCI    ExecEnv = 16384
+	ExecEnvEnableVhciInjection ExecEnv = 32768
+	ExecEnvEnableWifi          ExecEnv = 65536
+	ExecEnvDelayKcovMmap       ExecEnv = 131072
+	ExecEnvEnableNicVF         ExecEnv = 262144
 )
 
 var EnumNamesExecEnv = map[ExecEnv]string{
 	ExecEnvDebug:               "Debug",
 	ExecEnvSignal:              "Signal",
+	ExecEnvReadOnlyCoverage:    "ReadOnlyCoverage",
 	ExecEnvResetState:          "ResetState",
 	ExecEnvSandboxNone:         "SandboxNone",
 	ExecEnvSandboxSetuid:       "SandboxSetuid",
@@ -355,6 +360,7 @@ var EnumNamesExecEnv = map[ExecEnv]string{
 var EnumValuesExecEnv = map[string]ExecEnv{
 	"Debug":               ExecEnvDebug,
 	"Signal":              ExecEnvSignal,
+	"ReadOnlyCoverage":    ExecEnvReadOnlyCoverage,
 	"ResetState":          ExecEnvResetState,
 	"SandboxNone":         ExecEnvSandboxNone,
 	"SandboxSetuid":       ExecEnvSandboxSetuid,
diff --git a/pkg/flatrpc/flatrpc.h b/pkg/flatrpc/flatrpc.h
index def2648ad..6f7c181ec 100644
--- a/pkg/flatrpc/flatrpc.h
+++ b/pkg/flatrpc/flatrpc.h
@@ -141,34 +141,36 @@ enum class Feature : uint64_t {
   Comparisons = 2ULL,
   ExtraCoverage = 4ULL,
   DelayKcovMmap = 8ULL,
-  SandboxNone = 16ULL,
-  SandboxSetuid = 32ULL,
-  SandboxNamespace = 64ULL,
-  SandboxAndroid = 128ULL,
-  Fault = 256ULL,
-  Leak = 512ULL,
-  NetInjection = 1024ULL,
-  NetDevices = 2048ULL,
-  KCSAN = 4096ULL,
-  DevlinkPCI = 8192ULL,
-  NicVF = 16384ULL,
-  USBEmulation = 32768ULL,
-  VhciInjection = 65536ULL,
-  WifiEmulation = 131072ULL,
-  LRWPANEmulation = 262144ULL,
-  BinFmtMisc = 524288ULL,
-  Swap = 1048576ULL,
+  KcovResetIoctl = 16ULL,
+  SandboxNone = 32ULL,
+  SandboxSetuid = 64ULL,
+  SandboxNamespace = 128ULL,
+  SandboxAndroid = 256ULL,
+  Fault = 512ULL,
+  Leak = 1024ULL,
+  NetInjection = 2048ULL,
+  NetDevices = 4096ULL,
+  KCSAN = 8192ULL,
+  DevlinkPCI = 16384ULL,
+  NicVF = 32768ULL,
+  USBEmulation = 65536ULL,
+  VhciInjection = 131072ULL,
+  WifiEmulation = 262144ULL,
+  LRWPANEmulation = 524288ULL,
+  BinFmtMisc = 1048576ULL,
+  Swap = 2097152ULL,
   NONE = 0,
-  ANY = 2097151ULL
+  ANY = 4194303ULL
 };
 FLATBUFFERS_DEFINE_BITMASK_OPERATORS(Feature, uint64_t)
 
-inline const Feature (&EnumValuesFeature())[21] {
+inline const Feature (&EnumValuesFeature())[22] {
   static const Feature values[] = {
     Feature::Coverage,
     Feature::Comparisons,
     Feature::ExtraCoverage,
     Feature::DelayKcovMmap,
+    Feature::KcovResetIoctl,
     Feature::SandboxNone,
     Feature::SandboxSetuid,
     Feature::SandboxNamespace,
@@ -196,6 +198,7 @@ inline const char *EnumNameFeature(Feature e) {
     case Feature::Comparisons: return "Comparisons";
     case Feature::ExtraCoverage: return "ExtraCoverage";
     case Feature::DelayKcovMmap: return "DelayKcovMmap";
+    case Feature::KcovResetIoctl: return "KcovResetIoctl";
     case Feature::SandboxNone: return "SandboxNone";
     case Feature::SandboxSetuid: return "SandboxSetuid";
     case Feature::SandboxNamespace: return "SandboxNamespace";
@@ -557,31 +560,33 @@ inline const char *EnumNameRequestFlag(RequestFlag e) {
 enum class ExecEnv : uint64_t {
   Debug = 1ULL,
   Signal = 2ULL,
-  ResetState = 4ULL,
-  SandboxNone = 8ULL,
-  SandboxSetuid = 16ULL,
-  SandboxNamespace = 32ULL,
-  SandboxAndroid = 64ULL,
-  ExtraCover = 128ULL,
-  EnableTun = 256ULL,
-  EnableNetDev = 512ULL,
-  EnableNetReset = 1024ULL,
-  EnableCgroups = 2048ULL,
-  EnableCloseFds = 4096ULL,
-  EnableDevlinkPCI = 8192ULL,
-  EnableVhciInjection = 16384ULL,
-  EnableWifi = 32768ULL,
-  DelayKcovMmap = 65536ULL,
-  EnableNicVF = 131072ULL,
+  ReadOnlyCoverage = 4ULL,
+  ResetState = 8ULL,
+  SandboxNone = 16ULL,
+  SandboxSetuid = 32ULL,
+  SandboxNamespace = 64ULL,
+  SandboxAndroid = 128ULL,
+  ExtraCover = 256ULL,
+  EnableTun = 512ULL,
+  EnableNetDev = 1024ULL,
+  EnableNetReset = 2048ULL,
+  EnableCgroups = 4096ULL,
+  EnableCloseFds = 8192ULL,
+  EnableDevlinkPCI = 16384ULL,
+  EnableVhciInjection = 32768ULL,
+  EnableWifi = 65536ULL,
+  DelayKcovMmap = 131072ULL,
+  EnableNicVF = 262144ULL,
   NONE = 0,
-  ANY = 262143ULL
+  ANY = 524287ULL
 };
 FLATBUFFERS_DEFINE_BITMASK_OPERATORS(ExecEnv, uint64_t)
 
-inline const ExecEnv (&EnumValuesExecEnv())[18] {
+inline const ExecEnv (&EnumValuesExecEnv())[19] {
   static const ExecEnv values[] = {
     ExecEnv::Debug,
     ExecEnv::Signal,
+    ExecEnv::ReadOnlyCoverage,
     ExecEnv::ResetState,
     ExecEnv::SandboxNone,
     ExecEnv::SandboxSetuid,
@@ -606,6 +611,7 @@ inline const char *EnumNameExecEnv(ExecEnv e) {
   switch (e) {
     case ExecEnv::Debug: return "Debug";
     case ExecEnv::Signal: return "Signal";
+    case ExecEnv::ReadOnlyCoverage: return "ReadOnlyCoverage";
     case ExecEnv::ResetState: return "ResetState";
     case ExecEnv::SandboxNone: return "SandboxNone";
     case ExecEnv::SandboxSetuid: return "SandboxSetuid";
diff --git a/pkg/runtest/run_test.go b/pkg/runtest/run_test.go
index f37c7612c..b566f74dd 100644
--- a/pkg/runtest/run_test.go
+++ b/pkg/runtest/run_test.go
@@ -95,6 +95,7 @@ func test(t *testing.T, sysTarget *targets.Target) {
 			want := flatrpc.FeatureCoverage |
 				flatrpc.FeatureExtraCoverage |
 				flatrpc.FeatureDelayKcovMmap |
+				flatrpc.FeatureKcovResetIoctl |
 				flatrpc.FeatureSandboxNone |
 				flatrpc.FeatureFault |
 				flatrpc.FeatureNetDevices |
diff --git a/pkg/vminfo/features.go b/pkg/vminfo/features.go
index 2be067d32..6c4d6ae6d 100644
--- a/pkg/vminfo/features.go
+++ b/pkg/vminfo/features.go
@@ -138,6 +138,8 @@ func (ctx *checkContext) featureToFlags(feat flatrpc.Feature) (flatrpc.ExecEnv,
 	case flatrpc.FeatureDelayKcovMmap:
 		envFlags |= flatrpc.ExecEnvSignal | flatrpc.ExecEnvDelayKcovMmap
 		execFlags |= flatrpc.ExecFlagCollectSignal | flatrpc.ExecFlagCollectCover
+	case flatrpc.FeatureKcovResetIoctl:
+		envFlags |= flatrpc.ExecEnvReadOnlyCoverage
 	case flatrpc.FeatureSandboxNone:
 		envFlags &= ^ctx.cfg.Sandbox
 		envFlags |= flatrpc.ExecEnvSandboxNone