sys/openbsd: sanitize setrlimit(RLIMIT_DATA) syscalls (#1231)

OpenBSD performs a strict validation of the RLIMIT_DATA soft limit
during memory allocation. Lowering the same limit could cause
syz-executor to run out of memory quickly. Therefore make sure to not go
lower than the default soft limit for the staff group.

This is one of the root causes of the high amount of reported "lost
connection to test machine".

2 files changed, 35 insertions, 0 deletions

diff --git a/sys/openbsd/init.go b/sys/openbsd/init.go
index c42fe0497..8dabf3118 100644
--- a/sys/openbsd/init.go
+++ b/sys/openbsd/init.go
@@ -40,6 +40,9 @@ const (
 	kcovFdMinorMin = 232
 	// kOutPipeFd in executor/executor.cc
 	kcovFdMinorMax = 248
+
+	// RLIMIT_DATA from openbsd:src/sys/sys/resource.h
+	rlimitData = 2
 )
 
 // openbsd:src/sys/sys/types.h
@@ -106,6 +109,28 @@ func (arch *arch) SanitizeCall(c *prog.Call) {
 		if devmajor(dev.Val) == 4 && devminor(dev.Val) == 2 {
 			dev.Val = devNullDevT
 		}
+	case "setrlimit":
+		// OpenBSD performs a strict validation of the RLIMIT_DATA soft
+		// limit during memory allocation. Lowering the same limit could
+		// cause syz-executor to run out of memory quickly. Therefore
+		// make sure to not go lower than the default soft limit for the
+		// staff group.
+		if c.Args[0].(*prog.ConstArg).Val != rlimitData {
+			break
+		}
+		var rlimitDataMin uint64 = 1536 * 1024 * 1024
+		ptr := c.Args[1].(*prog.PointerArg)
+		if ptr.Res != nil {
+			args := ptr.Res.(*prog.GroupArg).Inner
+			for _, arg := range args {
+				switch v := arg.(type) {
+				case *prog.ConstArg:
+					if v.Val < rlimitDataMin {
+						v.Val = rlimitDataMin
+					}
+				}
+			}
+		}
 	default:
 		arch.unix.SanitizeCall(c)
 	}
diff --git a/sys/openbsd/init_test.go b/sys/openbsd/init_test.go
index bbd93d6a7..157b46a86 100644
--- a/sys/openbsd/init_test.go
+++ b/sys/openbsd/init_test.go
@@ -46,6 +46,16 @@ func TestSanitizeMknodCall(t *testing.T) {
 			`mknod(0x0, 0x0, 0x0402)`,
 			`mknod(0x0, 0x0, 0x202)`,
 		},
+		{
+			// RLIMIT_DATA
+			`setrlimit(0x2, &(0x7f0000cc0ff0)={0x0, 0x80000000})`,
+			`setrlimit(0x2, &(0x7f0000cc0ff0)={0x60000000, 0x80000000})`,
+		},
+		{
+			// RLIMIT_CPU
+			`setrlimit(0x0, &(0x7f0000cc0ff0)={0x1, 0x1})`,
+			`setrlimit(0x0, &(0x7f0000cc0ff0)={0x1, 0x1})`,
+		},
 	}
 	for i, test := range tests {
 		t.Run(fmt.Sprint(i), func(t *testing.T) {