From 8304907db07a4c6f7ef3164be77c1bacc94d6d51 Mon Sep 17 00:00:00 2001
From: Dmitry Vyukov <dvyukov@google.com>
Date: Fri, 19 Jul 2019 09:51:07 +0200
Subject: tools/syz-env: restrict Makefile parallelism based on RAM

Ensure that we have at least 1GB per Makefile job.
Go compiler/linker can consume significant amount of memory
(observed to consume at least 600MB). See #1276 for context.
Update #1276
---
 pkg/osutil/osutil_akaros.go  |  4 ++++
 pkg/osutil/osutil_bsd.go     |  4 ++++
 pkg/osutil/osutil_darwin.go  |  4 ++++
 pkg/osutil/osutil_fuchsia.go |  4 ++++
 pkg/osutil/osutil_linux.go   |  6 ++++++
 pkg/osutil/osutil_windows.go |  4 ++++
 tools/syz-env/env.go         | 13 ++++++++++++-
 7 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/pkg/osutil/osutil_akaros.go b/pkg/osutil/osutil_akaros.go
index bcc285748..bc5b68b6a 100644
--- a/pkg/osutil/osutil_akaros.go
+++ b/pkg/osutil/osutil_akaros.go
@@ -17,6 +17,10 @@ func RemoveAll(dir string) error {
 	return os.RemoveAll(dir)
 }
 
+func SystemMemorySize() uint64 {
+	return 0
+}
+
 func prolongPipe(r, w *os.File) {
 }
 
diff --git a/pkg/osutil/osutil_bsd.go b/pkg/osutil/osutil_bsd.go
index 1f5cf6be8..bf97ae73b 100644
--- a/pkg/osutil/osutil_bsd.go
+++ b/pkg/osutil/osutil_bsd.go
@@ -14,6 +14,10 @@ func RemoveAll(dir string) error {
 	return os.RemoveAll(dir)
 }
 
+func SystemMemorySize() uint64 {
+	return 0
+}
+
 func prolongPipe(r, w *os.File) {
 }
 
diff --git a/pkg/osutil/osutil_darwin.go b/pkg/osutil/osutil_darwin.go
index bf6ba9810..68cd81da6 100644
--- a/pkg/osutil/osutil_darwin.go
+++ b/pkg/osutil/osutil_darwin.go
@@ -14,6 +14,10 @@ func RemoveAll(dir string) error {
 	return os.RemoveAll(dir)
 }
 
+func SystemMemorySize() uint64 {
+	return 0
+}
+
 func prolongPipe(r, w *os.File) {
 }
 
diff --git a/pkg/osutil/osutil_fuchsia.go b/pkg/osutil/osutil_fuchsia.go
index dc0ad5ddc..6ef2b08b2 100644
--- a/pkg/osutil/osutil_fuchsia.go
+++ b/pkg/osutil/osutil_fuchsia.go
@@ -18,6 +18,10 @@ func RemoveAll(dir string) error {
 	return os.RemoveAll(dir)
 }
 
+func SystemMemorySize() uint64 {
+	return 0
+}
+
 func CreateMemMappedFile(size int) (f *os.File, mem []byte, err error) {
 	return nil, nil, fmt.Errorf("CreateMemMappedFile is not implemented")
 }
diff --git a/pkg/osutil/osutil_linux.go b/pkg/osutil/osutil_linux.go
index 732d8e4c2..eee0a9009 100644
--- a/pkg/osutil/osutil_linux.go
+++ b/pkg/osutil/osutil_linux.go
@@ -38,6 +38,12 @@ func RemoveAll(dir string) error {
 	return nil
 }
 
+func SystemMemorySize() uint64 {
+	var info syscall.Sysinfo_t
+	syscall.Sysinfo(&info)
+	return info.Totalram
+}
+
 func removeImmutable(fname string) error {
 	// Reset FS_XFLAG_IMMUTABLE/FS_XFLAG_APPEND.
 	fd, err := syscall.Open(fname, syscall.O_RDONLY, 0)
diff --git a/pkg/osutil/osutil_windows.go b/pkg/osutil/osutil_windows.go
index a7cdb1c4f..d98a9d33c 100644
--- a/pkg/osutil/osutil_windows.go
+++ b/pkg/osutil/osutil_windows.go
@@ -19,6 +19,10 @@ func RemoveAll(dir string) error {
 	return os.RemoveAll(dir)
 }
 
+func SystemMemorySize() uint64 {
+	return 0
+}
+
 func prolongPipe(r, w *os.File) {
 }
 
diff --git a/tools/syz-env/env.go b/tools/syz-env/env.go
index cae88c5a9..aa0143ab9 100644
--- a/tools/syz-env/env.go
+++ b/tools/syz-env/env.go
@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/google/syzkaller/pkg/osutil"
 	"github.com/google/syzkaller/sys/targets"
 )
 
@@ -29,6 +30,16 @@ func main() {
 		Name string
 		Val  string
 	}
+	parallelism := runtime.NumCPU()
+	if mem := osutil.SystemMemorySize(); mem != 0 {
+		// Ensure that we have at least 1GB per Makefile job.
+		// Go compiler/linker can consume significant amount of memory
+		// (observed to consume at least 600MB). See #1276 for context.
+		memLimit := int(mem / (1 << 30))
+		if parallelism > memLimit {
+			parallelism = memLimit
+		}
+	}
 	vars := []Var{
 		{"BUILDOS", runtime.GOOS},
 		{"NATIVEBUILDOS", target.BuildOS},
@@ -39,7 +50,7 @@ func main() {
 		{"TARGETVMARCH", targetVMArch},
 		{"CC", target.CCompiler},
 		{"ADDCFLAGS", strings.Join(target.CrossCFlags, " ")},
-		{"NCORES", strconv.Itoa(runtime.NumCPU())},
+		{"NCORES", strconv.Itoa(parallelism)},
 		{"EXE", target.ExeExtension},
 		{"NATIVEBUILDOS", target.BuildOS},
 	}
-- 
cgit mrf-deployment