sys/openbsd: avoid /dev/fd node creation

Prevent nodes that maps to an already open kcov fd from being created since
they can corrupt the coverage buffer.

Partial revert of commit 04aed72692137822b809098c55401dd3493dd0f6 with some
tweaks and testing.

2 files changed, 86 insertions, 5 deletions

diff --git a/sys/openbsd/init.go b/sys/openbsd/init.go
index 4dfb1a930..cb277ee3e 100644
--- a/sys/openbsd/init.go
+++ b/sys/openbsd/init.go
@@ -25,20 +25,51 @@ type arch struct {
 	S_IFCHR uint64
 }
 
+const (
+	mknodMode = 0
+	mknodDev  = 1
+
+	// openbsd:src/etc/etc.amd64/MAKEDEV
+	devFdMajor  = 22
+	devNullDevT = 0x0202
+
+	// kCoverFd in executor/executor.cc
+	kcovFdMinorMin = 232
+	// kOutPipeFd in executor/executor.cc
+	kcovFdMinorMax = 248
+)
+
+func isKcovFd(dev uint64) bool {
+	// openbsd:src/sys/sys/types.h
+	major := (dev >> 8) & 0xff
+	minor := (dev & 0xff) | ((dev & 0xffff0000) >> 8)
+
+	return major == devFdMajor && minor >= kcovFdMinorMin && minor < kcovFdMinorMax
+}
+
 func (arch *arch) SanitizeCall(c *prog.Call) {
-	// Prevent vnodes of type VBAD from being created. Such vnodes will
-	// likely trigger assertion errors by the kernel.
-	pos := 1
+	argStart := 1
 	switch c.Meta.CallName {
 	case "mknodat":
-		pos = 2
+		argStart = 2
 		fallthrough
 	case "mknod":
-		mode := c.Args[pos].(*prog.ConstArg)
+		// Prevent vnodes of type VBAD from being created. Such vnodes will
+		// likely trigger assertion errors by the kernel.
+		mode := c.Args[argStart+mknodMode].(*prog.ConstArg)
 		if mode.Val&arch.S_IFMT == arch.S_IFMT {
 			mode.Val &^= arch.S_IFMT
 			mode.Val |= arch.S_IFCHR
 		}
+
+		// Prevent /dev/fd/X devices from getting created where X maps
+		// to an open kcov fd. They interfere with kcov data collection
+		// and cause corpus explosion.
+		// https://groups.google.com/d/msg/syzkaller/_IRWeAjVoy4/Akl2XMZTDAAJ
+		dev := c.Args[argStart+mknodDev].(*prog.ConstArg)
+		if isKcovFd(dev.Val) {
+			dev.Val = devNullDevT
+		}
 	default:
 		arch.unix.SanitizeCall(c)
 	}
diff --git a/sys/openbsd/init_test.go b/sys/openbsd/init_test.go
new file mode 100644
index 000000000..780fe3efb
--- /dev/null
+++ b/sys/openbsd/init_test.go
@@ -0,0 +1,50 @@
+package openbsd_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/google/syzkaller/prog"
+	_ "github.com/google/syzkaller/sys/openbsd/gen"
+)
+
+func TestSanitizeMknodCall(t *testing.T) {
+	target, err := prog.GetTarget("openbsd", "amd64")
+	if err != nil {
+		t.Fatal(err)
+	}
+	tests := []struct {
+		input  string
+		output string
+	}{
+		{
+			// major=22, minor=232
+			`mknodat(0x0, 0x0, 0x0, 0x16e8)`,
+			`mknodat(0x0, 0x0, 0x0, 0x202)`,
+		},
+		{
+			// major=22, minor=232
+			`mknod(0x0, 0x0, 0x16e8)`,
+			`mknod(0x0, 0x0, 0x202)`,
+		},
+		{
+			// major=22, minor=0
+			`mknod(0x0, 0x0, 0x1600)`,
+			`mknod(0x0, 0x0, 0x1600)`,
+		},
+	}
+	for i, test := range tests {
+		t.Run(fmt.Sprint(i), func(t *testing.T) {
+			p, err := target.Deserialize([]byte(test.input), prog.Strict)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got := strings.TrimSpace(string(p.Serialize()))
+			want := strings.TrimSpace(test.output)
+			if got != want {
+				t.Fatalf("input:\n%v\ngot:\n%v\nwant:\n%s", test.input, got, want)
+			}
+		})
+	}
+}